# import the required libraries
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

# filter out the warnings
import warnings
warnings.filterwarnings('ignore')

# setting maximum rows & columns display size to 200 for better visibility of data 
pd.set_option('display.max_rows', 200)
pd.set_option('display.max_columns', 200)

# load dataset 
app_df =  pd.read_csv('application_data.csv')

# viewing first 5 rows
app_df.head(5)

#viewing last 5 rows
app_df.tail(5)

app_df.shape

(307511, 122)

app_df.columns.values

array(['SK_ID_CURR', 'TARGET', 'NAME_CONTRACT_TYPE', 'CODE_GENDER',
       'FLAG_OWN_CAR', 'FLAG_OWN_REALTY', 'CNT_CHILDREN',
       'AMT_INCOME_TOTAL', 'AMT_CREDIT', 'AMT_ANNUITY', 'AMT_GOODS_PRICE',
       'NAME_TYPE_SUITE', 'NAME_INCOME_TYPE', 'NAME_EDUCATION_TYPE',
       'NAME_FAMILY_STATUS', 'NAME_HOUSING_TYPE',
       'REGION_POPULATION_RELATIVE', 'DAYS_BIRTH', 'DAYS_EMPLOYED',
       'DAYS_REGISTRATION', 'DAYS_ID_PUBLISH', 'OWN_CAR_AGE',
       'FLAG_MOBIL', 'FLAG_EMP_PHONE', 'FLAG_WORK_PHONE',
       'FLAG_CONT_MOBILE', 'FLAG_PHONE', 'FLAG_EMAIL', 'OCCUPATION_TYPE',
       'CNT_FAM_MEMBERS', 'REGION_RATING_CLIENT',
       'REGION_RATING_CLIENT_W_CITY', 'WEEKDAY_APPR_PROCESS_START',
       'HOUR_APPR_PROCESS_START', 'REG_REGION_NOT_LIVE_REGION',
       'REG_REGION_NOT_WORK_REGION', 'LIVE_REGION_NOT_WORK_REGION',
       'REG_CITY_NOT_LIVE_CITY', 'REG_CITY_NOT_WORK_CITY',
       'LIVE_CITY_NOT_WORK_CITY', 'ORGANIZATION_TYPE', 'EXT_SOURCE_1',
       'EXT_SOURCE_2', 'EXT_SOURCE_3', 'APARTMENTS_AVG',
       'BASEMENTAREA_AVG', 'YEARS_BEGINEXPLUATATION_AVG',
       'YEARS_BUILD_AVG', 'COMMONAREA_AVG', 'ELEVATORS_AVG',
       'ENTRANCES_AVG', 'FLOORSMAX_AVG', 'FLOORSMIN_AVG', 'LANDAREA_AVG',
       'LIVINGAPARTMENTS_AVG', 'LIVINGAREA_AVG',
       'NONLIVINGAPARTMENTS_AVG', 'NONLIVINGAREA_AVG', 'APARTMENTS_MODE',
       'BASEMENTAREA_MODE', 'YEARS_BEGINEXPLUATATION_MODE',
       'YEARS_BUILD_MODE', 'COMMONAREA_MODE', 'ELEVATORS_MODE',
       'ENTRANCES_MODE', 'FLOORSMAX_MODE', 'FLOORSMIN_MODE',
       'LANDAREA_MODE', 'LIVINGAPARTMENTS_MODE', 'LIVINGAREA_MODE',
       'NONLIVINGAPARTMENTS_MODE', 'NONLIVINGAREA_MODE',
       'APARTMENTS_MEDI', 'BASEMENTAREA_MEDI',
       'YEARS_BEGINEXPLUATATION_MEDI', 'YEARS_BUILD_MEDI',
       'COMMONAREA_MEDI', 'ELEVATORS_MEDI', 'ENTRANCES_MEDI',
       'FLOORSMAX_MEDI', 'FLOORSMIN_MEDI', 'LANDAREA_MEDI',
       'LIVINGAPARTMENTS_MEDI', 'LIVINGAREA_MEDI',
       'NONLIVINGAPARTMENTS_MEDI', 'NONLIVINGAREA_MEDI',
       'FONDKAPREMONT_MODE', 'HOUSETYPE_MODE', 'TOTALAREA_MODE',
       'WALLSMATERIAL_MODE', 'EMERGENCYSTATE_MODE',
       'OBS_30_CNT_SOCIAL_CIRCLE', 'DEF_30_CNT_SOCIAL_CIRCLE',
       'OBS_60_CNT_SOCIAL_CIRCLE', 'DEF_60_CNT_SOCIAL_CIRCLE',
       'DAYS_LAST_PHONE_CHANGE', 'FLAG_DOCUMENT_2', 'FLAG_DOCUMENT_3',
       'FLAG_DOCUMENT_4', 'FLAG_DOCUMENT_5', 'FLAG_DOCUMENT_6',
       'FLAG_DOCUMENT_7', 'FLAG_DOCUMENT_8', 'FLAG_DOCUMENT_9',
       'FLAG_DOCUMENT_10', 'FLAG_DOCUMENT_11', 'FLAG_DOCUMENT_12',
       'FLAG_DOCUMENT_13', 'FLAG_DOCUMENT_14', 'FLAG_DOCUMENT_15',
       'FLAG_DOCUMENT_16', 'FLAG_DOCUMENT_17', 'FLAG_DOCUMENT_18',
       'FLAG_DOCUMENT_19', 'FLAG_DOCUMENT_20', 'FLAG_DOCUMENT_21',
       'AMT_REQ_CREDIT_BUREAU_HOUR', 'AMT_REQ_CREDIT_BUREAU_DAY',
       'AMT_REQ_CREDIT_BUREAU_WEEK', 'AMT_REQ_CREDIT_BUREAU_MON',
       'AMT_REQ_CREDIT_BUREAU_QRT', 'AMT_REQ_CREDIT_BUREAU_YEAR'],
      dtype=object)

# checking the data types of all the columns
app_df.dtypes

SK_ID_CURR                        int64
TARGET                            int64
NAME_CONTRACT_TYPE               object
CODE_GENDER                      object
FLAG_OWN_CAR                     object
FLAG_OWN_REALTY                  object
CNT_CHILDREN                      int64
AMT_INCOME_TOTAL                float64
AMT_CREDIT                      float64
AMT_ANNUITY                     float64
AMT_GOODS_PRICE                 float64
NAME_TYPE_SUITE                  object
NAME_INCOME_TYPE                 object
NAME_EDUCATION_TYPE              object
NAME_FAMILY_STATUS               object
NAME_HOUSING_TYPE                object
REGION_POPULATION_RELATIVE      float64
DAYS_BIRTH                        int64
DAYS_EMPLOYED                     int64
DAYS_REGISTRATION               float64
DAYS_ID_PUBLISH                   int64
OWN_CAR_AGE                     float64
FLAG_MOBIL                        int64
FLAG_EMP_PHONE                    int64
FLAG_WORK_PHONE                   int64
FLAG_CONT_MOBILE                  int64
FLAG_PHONE                        int64
FLAG_EMAIL                        int64
OCCUPATION_TYPE                  object
CNT_FAM_MEMBERS                 float64
REGION_RATING_CLIENT              int64
REGION_RATING_CLIENT_W_CITY       int64
WEEKDAY_APPR_PROCESS_START       object
HOUR_APPR_PROCESS_START           int64
REG_REGION_NOT_LIVE_REGION        int64
REG_REGION_NOT_WORK_REGION        int64
LIVE_REGION_NOT_WORK_REGION       int64
REG_CITY_NOT_LIVE_CITY            int64
REG_CITY_NOT_WORK_CITY            int64
LIVE_CITY_NOT_WORK_CITY           int64
ORGANIZATION_TYPE                object
EXT_SOURCE_1                    float64
EXT_SOURCE_2                    float64
EXT_SOURCE_3                    float64
APARTMENTS_AVG                  float64
BASEMENTAREA_AVG                float64
YEARS_BEGINEXPLUATATION_AVG     float64
YEARS_BUILD_AVG                 float64
COMMONAREA_AVG                  float64
ELEVATORS_AVG                   float64
ENTRANCES_AVG                   float64
FLOORSMAX_AVG                   float64
FLOORSMIN_AVG                   float64
LANDAREA_AVG                    float64
LIVINGAPARTMENTS_AVG            float64
LIVINGAREA_AVG                  float64
NONLIVINGAPARTMENTS_AVG         float64
NONLIVINGAREA_AVG               float64
APARTMENTS_MODE                 float64
BASEMENTAREA_MODE               float64
YEARS_BEGINEXPLUATATION_MODE    float64
YEARS_BUILD_MODE                float64
COMMONAREA_MODE                 float64
ELEVATORS_MODE                  float64
ENTRANCES_MODE                  float64
FLOORSMAX_MODE                  float64
FLOORSMIN_MODE                  float64
LANDAREA_MODE                   float64
LIVINGAPARTMENTS_MODE           float64
LIVINGAREA_MODE                 float64
NONLIVINGAPARTMENTS_MODE        float64
NONLIVINGAREA_MODE              float64
APARTMENTS_MEDI                 float64
BASEMENTAREA_MEDI               float64
YEARS_BEGINEXPLUATATION_MEDI    float64
YEARS_BUILD_MEDI                float64
COMMONAREA_MEDI                 float64
ELEVATORS_MEDI                  float64
ENTRANCES_MEDI                  float64
FLOORSMAX_MEDI                  float64
FLOORSMIN_MEDI                  float64
LANDAREA_MEDI                   float64
LIVINGAPARTMENTS_MEDI           float64
LIVINGAREA_MEDI                 float64
NONLIVINGAPARTMENTS_MEDI        float64
NONLIVINGAREA_MEDI              float64
FONDKAPREMONT_MODE               object
HOUSETYPE_MODE                   object
TOTALAREA_MODE                  float64
WALLSMATERIAL_MODE               object
EMERGENCYSTATE_MODE              object
OBS_30_CNT_SOCIAL_CIRCLE        float64
DEF_30_CNT_SOCIAL_CIRCLE        float64
OBS_60_CNT_SOCIAL_CIRCLE        float64
DEF_60_CNT_SOCIAL_CIRCLE        float64
DAYS_LAST_PHONE_CHANGE          float64
FLAG_DOCUMENT_2                   int64
FLAG_DOCUMENT_3                   int64
FLAG_DOCUMENT_4                   int64
FLAG_DOCUMENT_5                   int64
FLAG_DOCUMENT_6                   int64
FLAG_DOCUMENT_7                   int64
FLAG_DOCUMENT_8                   int64
FLAG_DOCUMENT_9                   int64
FLAG_DOCUMENT_10                  int64
FLAG_DOCUMENT_11                  int64
FLAG_DOCUMENT_12                  int64
FLAG_DOCUMENT_13                  int64
FLAG_DOCUMENT_14                  int64
FLAG_DOCUMENT_15                  int64
FLAG_DOCUMENT_16                  int64
FLAG_DOCUMENT_17                  int64
FLAG_DOCUMENT_18                  int64
FLAG_DOCUMENT_19                  int64
FLAG_DOCUMENT_20                  int64
FLAG_DOCUMENT_21                  int64
AMT_REQ_CREDIT_BUREAU_HOUR      float64
AMT_REQ_CREDIT_BUREAU_DAY       float64
AMT_REQ_CREDIT_BUREAU_WEEK      float64
AMT_REQ_CREDIT_BUREAU_MON       float64
AMT_REQ_CREDIT_BUREAU_QRT       float64
AMT_REQ_CREDIT_BUREAU_YEAR      float64
dtype: object

# check the descriptive statistics of numerical variables
#convert to float for better visibility
pd.options.display.float_format = '{:,.2f}'.format
app_df.describe()

app_df['TARGET'].value_counts()

TARGET
0    282686
1     24825
Name: count, dtype: int64

app_df['TARGET'].value_counts()/len(app_df) * 100

TARGET
0   91.93
1    8.07
Name: count, dtype: float64

#set figure size
plt.figure(figsize=(4,3))
sns.countplot(data=app_df, x='TARGET')
plt.title('Count of TARGET variable per value')
plt.xlabel('Target Variable')
plt.ylabel('Count')
plt.show()

#concise summary of the dataframe to view data types of all columns & null values 
#to display all columns use verbose = true
app_df.info(verbose=True, memory_usage=True)

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 307511 entries, 0 to 307510
Data columns (total 122 columns):
 #    Column                        Dtype  
---   ------                        -----  
 0    SK_ID_CURR                    int64  
 1    TARGET                        int64  
 2    NAME_CONTRACT_TYPE            object 
 3    CODE_GENDER                   object 
 4    FLAG_OWN_CAR                  object 
 5    FLAG_OWN_REALTY               object 
 6    CNT_CHILDREN                  int64  
 7    AMT_INCOME_TOTAL              float64
 8    AMT_CREDIT                    float64
 9    AMT_ANNUITY                   float64
 10   AMT_GOODS_PRICE               float64
 11   NAME_TYPE_SUITE               object 
 12   NAME_INCOME_TYPE              object 
 13   NAME_EDUCATION_TYPE           object 
 14   NAME_FAMILY_STATUS            object 
 15   NAME_HOUSING_TYPE             object 
 16   REGION_POPULATION_RELATIVE    float64
 17   DAYS_BIRTH                    int64  
 18   DAYS_EMPLOYED                 int64  
 19   DAYS_REGISTRATION             float64
 20   DAYS_ID_PUBLISH               int64  
 21   OWN_CAR_AGE                   float64
 22   FLAG_MOBIL                    int64  
 23   FLAG_EMP_PHONE                int64  
 24   FLAG_WORK_PHONE               int64  
 25   FLAG_CONT_MOBILE              int64  
 26   FLAG_PHONE                    int64  
 27   FLAG_EMAIL                    int64  
 28   OCCUPATION_TYPE               object 
 29   CNT_FAM_MEMBERS               float64
 30   REGION_RATING_CLIENT          int64  
 31   REGION_RATING_CLIENT_W_CITY   int64  
 32   WEEKDAY_APPR_PROCESS_START    object 
 33   HOUR_APPR_PROCESS_START       int64  
 34   REG_REGION_NOT_LIVE_REGION    int64  
 35   REG_REGION_NOT_WORK_REGION    int64  
 36   LIVE_REGION_NOT_WORK_REGION   int64  
 37   REG_CITY_NOT_LIVE_CITY        int64  
 38   REG_CITY_NOT_WORK_CITY        int64  
 39   LIVE_CITY_NOT_WORK_CITY       int64  
 40   ORGANIZATION_TYPE             object 
 41   EXT_SOURCE_1                  float64
 42   EXT_SOURCE_2                  float64
 43   EXT_SOURCE_3                  float64
 44   APARTMENTS_AVG                float64
 45   BASEMENTAREA_AVG              float64
 46   YEARS_BEGINEXPLUATATION_AVG   float64
 47   YEARS_BUILD_AVG               float64
 48   COMMONAREA_AVG                float64
 49   ELEVATORS_AVG                 float64
 50   ENTRANCES_AVG                 float64
 51   FLOORSMAX_AVG                 float64
 52   FLOORSMIN_AVG                 float64
 53   LANDAREA_AVG                  float64
 54   LIVINGAPARTMENTS_AVG          float64
 55   LIVINGAREA_AVG                float64
 56   NONLIVINGAPARTMENTS_AVG       float64
 57   NONLIVINGAREA_AVG             float64
 58   APARTMENTS_MODE               float64
 59   BASEMENTAREA_MODE             float64
 60   YEARS_BEGINEXPLUATATION_MODE  float64
 61   YEARS_BUILD_MODE              float64
 62   COMMONAREA_MODE               float64
 63   ELEVATORS_MODE                float64
 64   ENTRANCES_MODE                float64
 65   FLOORSMAX_MODE                float64
 66   FLOORSMIN_MODE                float64
 67   LANDAREA_MODE                 float64
 68   LIVINGAPARTMENTS_MODE         float64
 69   LIVINGAREA_MODE               float64
 70   NONLIVINGAPARTMENTS_MODE      float64
 71   NONLIVINGAREA_MODE            float64
 72   APARTMENTS_MEDI               float64
 73   BASEMENTAREA_MEDI             float64
 74   YEARS_BEGINEXPLUATATION_MEDI  float64
 75   YEARS_BUILD_MEDI              float64
 76   COMMONAREA_MEDI               float64
 77   ELEVATORS_MEDI                float64
 78   ENTRANCES_MEDI                float64
 79   FLOORSMAX_MEDI                float64
 80   FLOORSMIN_MEDI                float64
 81   LANDAREA_MEDI                 float64
 82   LIVINGAPARTMENTS_MEDI         float64
 83   LIVINGAREA_MEDI               float64
 84   NONLIVINGAPARTMENTS_MEDI      float64
 85   NONLIVINGAREA_MEDI            float64
 86   FONDKAPREMONT_MODE            object 
 87   HOUSETYPE_MODE                object 
 88   TOTALAREA_MODE                float64
 89   WALLSMATERIAL_MODE            object 
 90   EMERGENCYSTATE_MODE           object 
 91   OBS_30_CNT_SOCIAL_CIRCLE      float64
 92   DEF_30_CNT_SOCIAL_CIRCLE      float64
 93   OBS_60_CNT_SOCIAL_CIRCLE      float64
 94   DEF_60_CNT_SOCIAL_CIRCLE      float64
 95   DAYS_LAST_PHONE_CHANGE        float64
 96   FLAG_DOCUMENT_2               int64  
 97   FLAG_DOCUMENT_3               int64  
 98   FLAG_DOCUMENT_4               int64  
 99   FLAG_DOCUMENT_5               int64  
 100  FLAG_DOCUMENT_6               int64  
 101  FLAG_DOCUMENT_7               int64  
 102  FLAG_DOCUMENT_8               int64  
 103  FLAG_DOCUMENT_9               int64  
 104  FLAG_DOCUMENT_10              int64  
 105  FLAG_DOCUMENT_11              int64  
 106  FLAG_DOCUMENT_12              int64  
 107  FLAG_DOCUMENT_13              int64  
 108  FLAG_DOCUMENT_14              int64  
 109  FLAG_DOCUMENT_15              int64  
 110  FLAG_DOCUMENT_16              int64  
 111  FLAG_DOCUMENT_17              int64  
 112  FLAG_DOCUMENT_18              int64  
 113  FLAG_DOCUMENT_19              int64  
 114  FLAG_DOCUMENT_20              int64  
 115  FLAG_DOCUMENT_21              int64  
 116  AMT_REQ_CREDIT_BUREAU_HOUR    float64
 117  AMT_REQ_CREDIT_BUREAU_DAY     float64
 118  AMT_REQ_CREDIT_BUREAU_WEEK    float64
 119  AMT_REQ_CREDIT_BUREAU_MON     float64
 120  AMT_REQ_CREDIT_BUREAU_QRT     float64
 121  AMT_REQ_CREDIT_BUREAU_YEAR    float64
dtypes: float64(65), int64(41), object(16)
memory usage: 286.2+ MB

#Count null values per column
null_count = app_df.isnull().sum()
#Total number of rows
total_count = app_df.shape[0]
# Calculate percentage of nulls and round to 2 decimal places
null_pct = ((null_count / total_count) * 100).round(2)
#Filter only columns with missing values (makes plot cleaner)
null_pct = null_pct[null_pct > 0].sort_values(ascending=False)

#Plot using pointplot
plt.figure(figsize=(16,5))
sns.pointplot(x=null_pct.index, y=null_pct.values)
plt.xticks(rotation=90)
plt.xlabel("Columns")
plt.ylabel("Missing Values (%)")
plt.title("Percentage of Null Values per Column")
plt.show()

##create a copy of base data for manipulation & processing 
new_app_df = app_df.copy()

#check the new dataframe
new_app_df.head(5)

#check for missing value in percentage
null_pct.sort_values(ascending=True)

AMT_GOODS_PRICE                 0.09
EXT_SOURCE_2                    0.21
DEF_30_CNT_SOCIAL_CIRCLE        0.33
OBS_30_CNT_SOCIAL_CIRCLE        0.33
OBS_60_CNT_SOCIAL_CIRCLE        0.33
DEF_60_CNT_SOCIAL_CIRCLE        0.33
NAME_TYPE_SUITE                 0.42
AMT_REQ_CREDIT_BUREAU_MON      13.50
AMT_REQ_CREDIT_BUREAU_WEEK     13.50
AMT_REQ_CREDIT_BUREAU_YEAR     13.50
AMT_REQ_CREDIT_BUREAU_HOUR     13.50
AMT_REQ_CREDIT_BUREAU_QRT      13.50
AMT_REQ_CREDIT_BUREAU_DAY      13.50
EXT_SOURCE_3                   19.83
OCCUPATION_TYPE                31.35
EMERGENCYSTATE_MODE            47.40
TOTALAREA_MODE                 48.27
YEARS_BEGINEXPLUATATION_MEDI   48.78
YEARS_BEGINEXPLUATATION_AVG    48.78
YEARS_BEGINEXPLUATATION_MODE   48.78
FLOORSMAX_MEDI                 49.76
FLOORSMAX_MODE                 49.76
FLOORSMAX_AVG                  49.76
HOUSETYPE_MODE                 50.18
LIVINGAREA_MODE                50.19
LIVINGAREA_MEDI                50.19
LIVINGAREA_AVG                 50.19
ENTRANCES_MODE                 50.35
ENTRANCES_MEDI                 50.35
ENTRANCES_AVG                  50.35
APARTMENTS_AVG                 50.75
APARTMENTS_MODE                50.75
APARTMENTS_MEDI                50.75
WALLSMATERIAL_MODE             50.84
ELEVATORS_AVG                  53.30
ELEVATORS_MODE                 53.30
ELEVATORS_MEDI                 53.30
NONLIVINGAREA_AVG              55.18
NONLIVINGAREA_MODE             55.18
NONLIVINGAREA_MEDI             55.18
EXT_SOURCE_1                   56.38
BASEMENTAREA_MEDI              58.52
BASEMENTAREA_AVG               58.52
BASEMENTAREA_MODE              58.52
LANDAREA_MODE                  59.38
LANDAREA_AVG                   59.38
LANDAREA_MEDI                  59.38
OWN_CAR_AGE                    65.99
YEARS_BUILD_MODE               66.50
YEARS_BUILD_AVG                66.50
YEARS_BUILD_MEDI               66.50
FLOORSMIN_AVG                  67.85
FLOORSMIN_MODE                 67.85
FLOORSMIN_MEDI                 67.85
LIVINGAPARTMENTS_MODE          68.35
LIVINGAPARTMENTS_AVG           68.35
LIVINGAPARTMENTS_MEDI          68.35
FONDKAPREMONT_MODE             68.39
NONLIVINGAPARTMENTS_MEDI       69.43
NONLIVINGAPARTMENTS_AVG        69.43
NONLIVINGAPARTMENTS_MODE       69.43
COMMONAREA_MEDI                69.87
COMMONAREA_AVG                 69.87
COMMONAREA_MODE                69.87
dtype: float64

# get ONLY column names with >50% missing
cols_50 = null_pct[null_pct >= 50].index.tolist()
print("Columns with >50% missing values:")
print(cols_50)

Columns with >50% missing values:
['COMMONAREA_AVG', 'COMMONAREA_MODE', 'COMMONAREA_MEDI', 'NONLIVINGAPARTMENTS_AVG', 'NONLIVINGAPARTMENTS_MODE', 'NONLIVINGAPARTMENTS_MEDI', 'FONDKAPREMONT_MODE', 'LIVINGAPARTMENTS_MODE', 'LIVINGAPARTMENTS_AVG', 'LIVINGAPARTMENTS_MEDI', 'FLOORSMIN_MEDI', 'FLOORSMIN_AVG', 'FLOORSMIN_MODE', 'YEARS_BUILD_MEDI', 'YEARS_BUILD_MODE', 'YEARS_BUILD_AVG', 'OWN_CAR_AGE', 'LANDAREA_MEDI', 'LANDAREA_MODE', 'LANDAREA_AVG', 'BASEMENTAREA_AVG', 'BASEMENTAREA_MODE', 'BASEMENTAREA_MEDI', 'EXT_SOURCE_1', 'NONLIVINGAREA_MODE', 'NONLIVINGAREA_MEDI', 'NONLIVINGAREA_AVG', 'ELEVATORS_MODE', 'ELEVATORS_MEDI', 'ELEVATORS_AVG', 'WALLSMATERIAL_MODE', 'APARTMENTS_AVG', 'APARTMENTS_MODE', 'APARTMENTS_MEDI', 'ENTRANCES_AVG', 'ENTRANCES_MODE', 'ENTRANCES_MEDI', 'LIVINGAREA_AVG', 'LIVINGAREA_MODE', 'LIVINGAREA_MEDI', 'HOUSETYPE_MODE']

new_app_df.drop(cols_50, axis = 1, inplace = True)
new_app_df.shape

(307511, 81)

# get the column names with >40% missing

cols_between_40_and_50 = null_pct[(null_pct > 40) & (null_pct < 50)].index.tolist()
cols_between_40_and_50

['FLOORSMAX_AVG',
 'FLOORSMAX_MEDI',
 'FLOORSMAX_MODE',
 'YEARS_BEGINEXPLUATATION_AVG',
 'YEARS_BEGINEXPLUATATION_MODE',
 'YEARS_BEGINEXPLUATATION_MEDI',
 'TOTALAREA_MODE',
 'EMERGENCYSTATE_MODE']

new_app_df.drop(cols_between_40_and_50, axis=1, inplace=True)
new_app_df.shape

(307511, 114)

cat_cols = new_app_df.select_dtypes(include='object').columns

for col in cat_cols:
    if col in ['OCCUPATION_TYPE']:
        new_app_df[col].fillna('Unknown', inplace=True)
    else:
        new_app_df[col].fillna(new_app_df[col].mode()[0], inplace=True)

C:\Users\HP\AppData\Local\Temp\ipykernel_10208\2814705985.py:7: FutureWarning: A value is trying to be set on a copy of a DataFrame or Series through chained assignment using an inplace method.
The behavior will change in pandas 3.0. This inplace method will never work because the intermediate object on which we are setting values always behaves as a copy.

For example, when doing 'df[col].method(value, inplace=True)', try using 'df.method({col: value}, inplace=True)' or df[col] = df[col].method(value) instead, to perform the operation inplace on the original object.


  new_app_df[col].fillna(new_app_df[col].mode()[0], inplace=True)
C:\Users\HP\AppData\Local\Temp\ipykernel_10208\2814705985.py:5: FutureWarning: A value is trying to be set on a copy of a DataFrame or Series through chained assignment using an inplace method.
The behavior will change in pandas 3.0. This inplace method will never work because the intermediate object on which we are setting values always behaves as a copy.

For example, when doing 'df[col].method(value, inplace=True)', try using 'df.method({col: value}, inplace=True)' or df[col] = df[col].method(value) instead, to perform the operation inplace on the original object.


  new_app_df[col].fillna('Unknown', inplace=True)

num_cols = new_app_df.select_dtypes(include=['int64', 'float64']).columns
for col in num_cols:
    new_app_df[col].fillna(new_app_df[col].median(), inplace=True)

C:\Users\HP\AppData\Local\Temp\ipykernel_10208\3182007245.py:3: FutureWarning: A value is trying to be set on a copy of a DataFrame or Series through chained assignment using an inplace method.
The behavior will change in pandas 3.0. This inplace method will never work because the intermediate object on which we are setting values always behaves as a copy.

For example, when doing 'df[col].method(value, inplace=True)', try using 'df.method({col: value}, inplace=True)' or df[col] = df[col].method(value) instead, to perform the operation inplace on the original object.


  new_app_df[col].fillna(new_app_df[col].median(), inplace=True)

#sanity check to see if there are any null columns left
null_col_check = new_app_df.columns.isna().sum()
null_col_check

np.int64(0)

# fix FLAG columns (Y/N → 1/0)
flag_text_cols = ['FLAG_OWN_CAR', 'FLAG_OWN_REALTY']
for col in flag_text_cols:
    new_app_df[col] = new_app_df[col].map({'Y': 1, 'N': 0}).astype('int64')

count_cols = [
    'CNT_FAM_MEMBERS',
    'OBS_30_CNT_SOCIAL_CIRCLE', 'DEF_30_CNT_SOCIAL_CIRCLE',
    'OBS_60_CNT_SOCIAL_CIRCLE', 'DEF_60_CNT_SOCIAL_CIRCLE',
    'AMT_REQ_CREDIT_BUREAU_HOUR', 'AMT_REQ_CREDIT_BUREAU_DAY',
    'AMT_REQ_CREDIT_BUREAU_WEEK', 'AMT_REQ_CREDIT_BUREAU_MON',
    'AMT_REQ_CREDIT_BUREAU_QRT', 'AMT_REQ_CREDIT_BUREAU_YEAR'
]
for col in count_cols:
    new_app_df[col] = new_app_df[col].astype('int64')

days_cols = ['DAYS_BIRTH','DAYS_EMPLOYED','DAYS_REGISTRATION','DAYS_ID_PUBLISH','DAYS_LAST_PHONE_CHANGE']

for col in days_cols:
    new_app_df[col] = new_app_df[col].astype('int64')

pos_days_cols = ['DAYS_BIRTH','DAYS_EMPLOYED','DAYS_REGISTRATION','DAYS_ID_PUBLISH','DAYS_LAST_PHONE_CHANGE']
# concvert to absolute values
for col in pos_days_cols:
    new_app_df[col]= new_app_df[col].abs()

new_app_df['AGE_YEARS']= (new_app_df['DAYS_BIRTH'] / 365.25).round().astype('int64')
new_app_df['AGE_YEARS']

0         26
1         46
2         52
3         52
4         55
          ..
307506    26
307507    57
307508    41
307509    33
307510    46
Name: AGE_YEARS, Length: 307511, dtype: int64

new_app_df['YEARS_EMPLOYED'] = (new_app_df['DAYS_EMPLOYED'] / 365.25).round().astype('int64')
new_app_df['YEARS_EMPLOYED']

0            2
1            3
2            1
3            8
4            8
          ... 
307506       1
307507    1000
307508      22
307509      13
307510       3
Name: YEARS_EMPLOYED, Length: 307511, dtype: int64

#drop DAYS_BIRTH to avoid redundancy
new_app_df.drop(columns=['DAYS_BIRTH'], inplace=True)
new_app_df.drop(columns=['DAYS_EMPLOYED'], inplace=True)

new_app_df['YEARS_EMPLOYED'].replace(1000, np.nan, inplace=True)

C:\Users\HP\AppData\Local\Temp\ipykernel_10208\209312331.py:1: FutureWarning: A value is trying to be set on a copy of a DataFrame or Series through chained assignment using an inplace method.
The behavior will change in pandas 3.0. This inplace method will never work because the intermediate object on which we are setting values always behaves as a copy.

For example, when doing 'df[col].method(value, inplace=True)', try using 'df.method({col: value}, inplace=True)' or df[col] = df[col].method(value) instead, to perform the operation inplace on the original object.


  new_app_df['YEARS_EMPLOYED'].replace(1000, np.nan, inplace=True)

new_app_df['CODE_GENDER'].value_counts()

CODE_GENDER
F      202448
M      105059
XNA         4
Name: count, dtype: int64

new_app_df['CODE_GENDER'] = new_app_df['CODE_GENDER'].replace('XNA',np.nan)
new_app_df['CODE_GENDER'].value_counts()

CODE_GENDER
F    202448
M    105059
Name: count, dtype: int64

outlier_cols = [
    'AMT_INCOME_TOTAL', 'AMT_CREDIT', 'AMT_ANNUITY', 'AMT_GOODS_PRICE',
    'AGE_YEARS', 'YEARS_EMPLOYED', 'CNT_CHILDREN', 'CNT_FAM_MEMBERS',
    'EXT_SOURCE_2', 'EXT_SOURCE_3'
]

def find_outliers(df, col):
    Q1 = df[col].quantile(0.25)
    Q3 = df[col].quantile(0.75)
    IQR = Q3 - Q1
    lower = Q1 - 1.5 * IQR
    upper = Q3 + 1.5 * IQR
    outliers = df[(df[col] < lower) | (df[col] > upper)]
    return len(outliers), lower, upper

# Summary
outlier_summary = []
for col in outlier_cols:
    n_out, low, up = find_outliers(new_app_df, col)
    pct = n_out / len(new_app_df) * 100
    outlier_summary.append({
        'Column': col,
        'Outliers': n_out,
        '%': round(pct, 2),
        'Lower': round(low, 2),
        'Upper': round(up, 2)
    })
    
summary_df = pd.DataFrame(outlier_summary)
print(summary_df)

             Column  Outliers     %      Lower       Upper
0  AMT_INCOME_TOTAL     14035  4.56  -22500.00   337500.00
1        AMT_CREDIT      6562  2.13 -537975.00  1616625.00
2       AMT_ANNUITY      7504  2.44  -10584.00    61704.00
3   AMT_GOODS_PRICE     14728  4.79 -423000.00  1341000.00
4         AGE_YEARS         0  0.00       4.00       84.00
5    YEARS_EMPLOYED     13457  4.38      -8.50       19.50
6      CNT_CHILDREN      4272  1.39      -1.50        2.50
7   CNT_FAM_MEMBERS      4007  1.30       0.50        4.50
8      EXT_SOURCE_2         0  0.00      -0.01        1.07
9      EXT_SOURCE_3      4313  1.40       0.09        0.97

fig, axes = plt.subplots(2, 5, figsize=(18, 8))
axes = axes.ravel()

for i, col in enumerate(outlier_cols):
    sns.boxplot(data=new_app_df, y=col, ax=axes[i], color='lightblue')
    axes[i].set_title(col)
    axes[i].grid(True, alpha=0.3)

plt.tight_layout()
plt.show()

#capping
# List of columns to cap
cap_cols = [
    'AMT_INCOME_TOTAL', 'AMT_CREDIT', 'AMT_ANNUITY', 'AMT_GOODS_PRICE',
    'CNT_CHILDREN', 'CNT_FAM_MEMBERS','YEARS_EMPLOYED'
]

# Cap using IQR upper bound
for col in cap_cols:
    Q1 = new_app_df[col].quantile(0.25)
    Q3 = new_app_df[col].quantile(0.75)
    IQR = Q3 - Q1
    upper = Q3 + 1.5 * IQR
    new_app_df[col] = new_app_df[col].clip(upper=upper)

import seaborn as sns
import matplotlib.pyplot as plt

fig, axes = plt.subplots(2, 5, figsize=(18, 8))
axes = axes.ravel()

for i, col in enumerate(outlier_cols):
    sns.boxplot(data=new_app_df, y=col, ax=axes[i], color='lightblue')
    axes[i].set_title(col)
    axes[i].grid(True, alpha=0.3)

plt.tight_layout()
plt.show()

# Define the number of quantiles/bins (e.g., 4 quartiles)
N_BINS = 4 

# Create the new binned column
new_app_df['CREDIT_BIN_FREQ'] = pd.qcut(
    new_app_df['AMT_CREDIT'], 
    q=N_BINS, 
    labels=['Q1_Low', 'Q2_MidLow', 'Q3_MidHigh', 'Q4_High'], 
    duplicates='drop' # Handles cases where values are identical at the cutoff
)

new_app_df['CREDIT_BIN_FREQ'].value_counts().sort_index()

CREDIT_BIN_FREQ
Q1_Low        78421
Q2_MidLow     75428
Q3_MidHigh    77786
Q4_High       75876
Name: count, dtype: int64

# Define the number of quantiles/bins (e.g., 4 quartiles)
N_BINS = 4 

# Create the new binned column
new_app_df['ANNUITY_BIN_FREQ'] = pd.qcut(
    new_app_df['AMT_ANNUITY'], 
    q=N_BINS, 
    labels=['Q1_Low', 'Q2_MidLow', 'Q3_MidHigh', 'Q4_High'], 
    duplicates='drop' # Handles cases where values are identical at the cutoff
)

new_app_df['ANNUITY_BIN_FREQ'].value_counts().sort_index()

ANNUITY_BIN_FREQ
Q1_Low        76893
Q2_MidLow     76892
Q3_MidHigh    76962
Q4_High       76764
Name: count, dtype: int64

# Define the number of quantiles/bins (e.g., 4 quartiles)
N_BINS = 4 

# Create the new binned column
new_app_df['INCOME_BIN_FREQ'] = pd.qcut(
    new_app_df['AMT_INCOME_TOTAL'], 
    q=N_BINS, 
    labels=['Q1_Low', 'Q2_MidLow', 'Q3_MidHigh', 'Q4_High'], 
    duplicates='drop' # Handles cases where values are identical at the cutoff
)

new_app_df['INCOME_BIN_FREQ'].value_counts().sort_index()

INCOME_BIN_FREQ
Q1_Low        100578
Q2_MidLow      53182
Q3_MidHigh     82213
Q4_High        71538
Name: count, dtype: int64

# Define the number of quantiles/bins (e.g., 4 quartiles)
N_BINS = 4 

# Create the new binned column
new_app_df['GOODS_PRICE_BIN_FREQ'] = pd.qcut(
    new_app_df['AMT_GOODS_PRICE'], 
    q=N_BINS, 
    labels=['Q1_Low', 'Q2_MidLow', 'Q3_MidHigh', 'Q4_High'], 
    duplicates='drop' # Handles cases where values are identical at the cutoff
)

new_app_df['GOODS_PRICE_BIN_FREQ'].value_counts().sort_index()

GOODS_PRICE_BIN_FREQ
Q1_Low        79877
Q2_MidLow     78380
Q3_MidHigh    73720
Q4_High       75534
Name: count, dtype: int64

# Define the number of quantiles/bins (e.g., 4 quartiles)
N_BINS = 4

# Create the new binned column for Age
new_app_df['AGE_BIN_FREQ'] = pd.qcut(
    new_app_df['AGE_YEARS'], 
    q=N_BINS, 
    labels=['Q1_Youngest', 'Q2_Young', 'Q3_MidAge', 'Q4_Oldest'], 
    duplicates='drop' # Handles identical values at cutoffs
)
new_app_df['AGE_BIN_FREQ'].value_counts().sort_index()

AGE_BIN_FREQ
Q1_Youngest    80804
Q2_Young       76153
Q3_MidAge      78242
Q4_Oldest      72312
Name: count, dtype: int64

# Define the number of quantiles/bins (e.g., 4 quartiles)
N_BINS = 4

# Create the new binned column for Years Employed
new_app_df['EMPLOYMENT_BIN_FREQ'] = pd.qcut(
    new_app_df['YEARS_EMPLOYED'], 
    q=N_BINS, 
    labels=['Q1_Shortest', 'Q2_Short', 'Q3_Long', 'Q4_Longest'], 
    duplicates='drop' 
)
new_app_df['EMPLOYMENT_BIN_FREQ'].value_counts().sort_index()

EMPLOYMENT_BIN_FREQ
Q1_Shortest    75163
Q2_Short       69813
Q3_Long        51804
Q4_Longest     55357
Name: count, dtype: int64

# 1. FIX: Convert the column from 'category' type to generic 'object' (string) type.
# This removes the fixed category list constraint, allowing the new string to be added.
new_app_df['EMPLOYMENT_BIN_FREQ'] = new_app_df['EMPLOYMENT_BIN_FREQ'].astype(object)

print(new_app_df['EMPLOYMENT_BIN_FREQ'].isna().sum())

55374

new_app_df['EMPLOYMENT_BIN_FREQ'] = new_app_df['EMPLOYMENT_BIN_FREQ'].fillna('No Employment')

new_app_df['EMPLOYMENT_BIN_FREQ'].isna().sum()

np.int64(0)

new_app_df['CNT_CHILDREN'] = np.where(
    (new_app_df['CNT_CHILDREN'] > 2.49) & (new_app_df['CNT_CHILDREN'] < 2.51),
    3.0,
    new_app_df['CNT_CHILDREN']
)

# Enforce integer type now that the fix is applied
new_app_df['CNT_CHILDREN'] = new_app_df['CNT_CHILDREN'].astype(int)

print("--- Data Integrity Check (After Fixing 2.5 -> 3) ---")
print("Unique Counts in the MODIFIED original column:")
print(new_app_df['CNT_CHILDREN'].value_counts().sort_index())

##--- B. CREATE STRING TIER (FOR REPORTING/EDA) ---
##This converts the placeholder 3 into the readable string '3+ Children'.

CAP_VALUE = 3

new_app_df['CNT_CHILDREN_OBJ'] = np.where(
    new_app_df['CNT_CHILDREN'] == CAP_VALUE,
    f'{CAP_VALUE}+', # Labels the placeholder 3 as the string '3+ Children'
    new_app_df['CNT_CHILDREN'].astype(str) # Converts 0, 1, 2 to '0', '1', '2'
)
print(new_app_df['CNT_CHILDREN_OBJ'].value_counts().sort_index())

--- Data Integrity Check (After Fixing 2.5 -> 3) ---
Unique Counts in the MODIFIED original column:
CNT_CHILDREN
0    215371
1     61119
2     26749
3      4272
Name: count, dtype: int64
CNT_CHILDREN_OBJ
0     215371
1      61119
2      26749
3+      4272
Name: count, dtype: int64

# 1. FIX CORRUPTION (4.5 -> 5.0)
# We assume a similar corruption exists for family members (e.g., 4.5) and fix it.
new_app_df['CNT_FAM_MEMBERS'] = np.where(
    (new_app_df['CNT_FAM_MEMBERS'] > 4.49) & (new_app_df['CNT_FAM_MEMBERS'] < 4.51),
    5.0, # Fix 4.5 to 5.0
    new_app_df['CNT_FAM_MEMBERS']
)
new_app_df['CNT_FAM_MEMBERS'] = new_app_df['CNT_FAM_MEMBERS'].astype(int)

CAP_MEMBERS = 5

# 2. CREATE FINAL STRING TIER (Capping Logic)
# We cap all counts >= 5 into the '5+ Members' group.
new_app_df['CNT_FAM_MEMBERS_OBJ'] = np.where(
    new_app_df['CNT_FAM_MEMBERS'] >= CAP_MEMBERS,
    f'{CAP_MEMBERS}+', # Labels 5, 6, 7, etc. as '5+ Members'
    new_app_df['CNT_FAM_MEMBERS'].astype(str) # Converts 1, 2, 3, 4 to strings
)

new_app_df['CNT_FAM_MEMBERS_OBJ'].value_counts().sort_index()

CNT_FAM_MEMBERS_OBJ
1      67847
2     158359
3      52601
4      24697
5+      4007
Name: count, dtype: int64

# Define the number of quantiles/bins (e.g., 4 quartiles)
N_BINS = 4

# Create the new binned column for Age
new_app_df['EXT_SOURCE_2_BIN_FREQ'] = pd.qcut(
    new_app_df['EXT_SOURCE_2'], 
    q=N_BINS, 
    # Optional: name your categories 
    labels=['Q1_Low', 'Q2_MidLow', 'Q3_MidHigh', 'Q4_High'], 
    duplicates='drop' # Handles identical values at cutoffs
)
new_app_df['EXT_SOURCE_2_BIN_FREQ'].value_counts().sort_index()

EXT_SOURCE_2_BIN_FREQ
Q1_Low        76879
Q2_MidLow     77207
Q3_MidHigh    76548
Q4_High       76877
Name: count, dtype: int64

# Define the number of quantiles/bins (e.g., 4 quartiles)
N_BINS = 4

# Create the new binned column for Age
new_app_df['EXT_SOURCE_3_BIN_FREQ'] = pd.qcut(
    new_app_df['EXT_SOURCE_3'], 
    q=N_BINS, 
    labels=['Q1_Low', 'Q2_MidLow', 'Q3_MidHigh', 'Q4_High'], 
    duplicates='drop' # Handles identical values at cutoffs
)
new_app_df['EXT_SOURCE_3_BIN_FREQ'].value_counts().sort_index()

EXT_SOURCE_3_BIN_FREQ
Q1_Low         77497
Q2_MidLow     106845
Q3_MidHigh     47086
Q4_High        76083
Name: count, dtype: int64

#function to plot univariate categorical variables
def cat_plot(df, col_x):
    plt.figure(figsize=(10,5))
    # create the count plot
    ax = sns.countplot(data=df, x=col_x, hue='TARGET')

    # create the title
    plt.title(f"Target Distribution by {col_x}")
    plt.xticks(rotation=45, ha='right')
    #show the plot
    plt.show()

# 1. Calculate the Default Risk Rate (Mean of TARGET) for each category
# The mean of a binary column (0 or 1) is the proportion of 1s (defaulters)

def univariate_decision_support_risk_plot(df, col_x):
   default_rate_pct = df.groupby(col_x)['TARGET'].mean()

   # 2. Convert to percentage and round to 2 decimal places
   default_rate_pct = (default_rate_pct * 100).round(2)

   # 3. Sort the categories by risk rate (highest risk first)
   default_rate_pct = default_rate_pct.sort_values(ascending=False)

   # 4. Prepare data for plotting (reset index to make columns)
   plot_data = default_rate_pct.reset_index()
   plot_data.columns = [col_x, 'Default Risk Rate (%)']

   # Plot using pointplot (as requested)
   plt.figure(figsize=(10, 5))
   sns.barplot(x=col_x, y='Default Risk Rate (%)', data=plot_data)
   plt.xticks(rotation=45, ha='right')
   plt.xlabel(col_x)
   plt.ylabel("Default Risk Rate (%)")
   plt.title(f"Default Risk Rate by {col_x}")
   plt.show()

cat_plot(new_app_df,'NAME_CONTRACT_TYPE')

univariate_decision_support_risk_plot(new_app_df,'NAME_CONTRACT_TYPE')

cat_plot(new_app_df,'CODE_GENDER')

univariate_decision_support_risk_plot(new_app_df,'CODE_GENDER')

cat_plot(new_app_df,'OCCUPATION_TYPE')

univariate_decision_support_risk_plot(new_app_df,'OCCUPATION_TYPE')

cat_plot(new_app_df,'NAME_INCOME_TYPE')

univariate_decision_support_risk_plot(new_app_df,'NAME_INCOME_TYPE')

cat_plot(new_app_df,'NAME_EDUCATION_TYPE')

univariate_decision_support_risk_plot(new_app_df,'NAME_EDUCATION_TYPE')

cat_plot(new_app_df,'NAME_FAMILY_STATUS')

univariate_decision_support_risk_plot(new_app_df,'NAME_FAMILY_STATUS')

cat_plot(new_app_df,'NAME_HOUSING_TYPE')

univariate_decision_support_risk_plot(new_app_df,'NAME_HOUSING_TYPE')

cat_plot(new_app_df,'CNT_CHILDREN_OBJ')

univariate_decision_support_risk_plot(new_app_df,'CNT_CHILDREN_OBJ')

cat_plot(new_app_df,'CNT_FAM_MEMBERS_OBJ')

univariate_decision_support_risk_plot(new_app_df,'CNT_FAM_MEMBERS_OBJ')

cat_plot(new_app_df,'FLAG_OWN_CAR')

univariate_decision_support_risk_plot(new_app_df,'FLAG_OWN_CAR')

cat_plot(new_app_df,'FLAG_OWN_REALTY')

univariate_decision_support_risk_plot(new_app_df,'FLAG_OWN_REALTY')

cat_plot(new_app_df,'REGION_RATING_CLIENT')

univariate_decision_support_risk_plot(new_app_df,'REGION_RATING_CLIENT')

df0 = new_app_df[new_app_df['TARGET'] == 0]
df1 = new_app_df[new_app_df['TARGET'] == 1]

# function to plot numerical variables
def cont_plot(df, col_x):
    plt.figure(figsize=(10,6))
    
    # 1. Use plt.gca() to get the current Axes object for the first plot
    #    and assign it to 'ax'.
    ax = sns.kdeplot(
        df0[col_x], 
        label="Target 0 (No Default)", 
        color="Red",
        fill=True
    )
    
    # 2. Plot the second distribution, referencing the same axes object 'ax'
    #    to ensure they are overlaid correctly.
    ax = sns.kdeplot(
        df1[col_x], 
        label="Target 1 (Default)", 
        color="Blue",
        ax=ax, 
        fill=True
    )
    
    # 3. Create the title and labels
    plt.title(f"Target Distribution by {col_x}")
    plt.xlabel(col_x)
    plt.ylabel("Density")
    plt.legend()
    
    # 4. Show the plot
    plt.show()

cont_plot(new_app_df,'AMT_INCOME_TOTAL')

univariate_decision_support_risk_plot(new_app_df,'INCOME_BIN_FREQ')

cont_plot(new_app_df,'AMT_CREDIT')

univariate_decision_support_risk_plot(new_app_df,'CREDIT_BIN_FREQ')

cont_plot(new_app_df,'AMT_ANNUITY')

univariate_decision_support_risk_plot(new_app_df,'ANNUITY_BIN_FREQ')

cont_plot(new_app_df,'AMT_GOODS_PRICE')

univariate_decision_support_risk_plot(new_app_df,'GOODS_PRICE_BIN_FREQ')

cont_plot(new_app_df,'AGE_YEARS')

univariate_decision_support_risk_plot(new_app_df,'AGE_BIN_FREQ')

cont_plot(new_app_df,'YEARS_EMPLOYED')

univariate_decision_support_risk_plot(new_app_df,'EMPLOYMENT_BIN_FREQ')

cont_plot(new_app_df,'EXT_SOURCE_2')

univariate_decision_support_risk_plot(new_app_df,'EXT_SOURCE_2_BIN_FREQ')

cont_plot(new_app_df,'EXT_SOURCE_3')

univariate_decision_support_risk_plot(new_app_df,'EXT_SOURCE_3_BIN_FREQ')

df0 = new_app_df[new_app_df['TARGET']==0]
df1 = new_app_df[new_app_df['TARGET']==1]

# function to plot categorical,categorical 
def cat_cat_plot(df,col_x,col_hue):

    num_hues = df[col_hue].nunique(dropna=False) 

    # 2. Generate the dynamic, high-contrast palette
    if num_hues <= 10:
        # Use 'Set1' for up to 9 categories for maximum contrast
        dynamic_palette = 'Set1' 
    elif num_hues <= 20:
        # Use 'tab20' for up to 20 categories
        dynamic_palette = sns.color_palette("tab20", n_colors=num_hues)
    elif num_hues > 20:
        # If there are more than 20, we use a different qualitative map like 'hls'
        # WARNING: If num_hues > 20, consider re-binning var2 as visualization will be cluttered.
        dynamic_palette = sns.color_palette("hls", n_colors=num_hues)
    
    
    plt.figure(figsize=(15,8))
    # --- Subplot 1: DEFAULTERS (df1) ---
    plt.subplot(1, 2, 1)
    # Pass the column names as strings (var1, var2) to sns.countplot
    ax1 = sns.countplot(data=df1, x=col_x, hue=col_hue, palette=dynamic_palette)

    # Pass the column names as strings for the title and labels
    plt.title(f"Distribution of {col_x} by {col_hue} for DEFAULTERS")
    plt.xlabel(col_x)
    plt.ylabel("Count")
    plt.xticks(rotation=45, ha='right')
    plt.legend(title=col_hue) # Added legend

    # --- Subplot 2: NON-DEFAULTERS (df0) ---
    plt.subplot(1, 2, 2)
    ax2 = sns.countplot(data=df0, x=col_x, hue=col_hue, palette=dynamic_palette)

    plt.title(f"Distribution of {col_x} by {col_hue} for Others")
    plt.xlabel(col_x)
    # Set ylabel to blank or remove it, as it's the same as the first plot's Y-axis
    plt.ylabel("Count") 
    plt.xticks(rotation=45, ha='right')
    plt.legend(title=col_hue) # Added legend
    
    # Crucial for preventing subplot titles and labels from overlapping
    plt.tight_layout()
    plt.show()

def bivariate_decision_support_risk_plot(df, col_x, col_hue):
    # 1. Group by both columns and calculate the mean of TARGET (the Default Rate)
    default_rate_pct = df.groupby([col_x, col_hue])['TARGET'].mean()
    default_rate_pct = (default_rate_pct * 100).round(2)

    plot_data = default_rate_pct.reset_index()
    plot_data.columns = [col_x, col_hue, 'Default Risk Rate (%)']

    num_hues = df[col_hue].nunique(dropna=False) 

    # 2. Generate the dynamic, high-contrast palette
    if num_hues <= 10:
        # Use 'Set1' for up to 9 categories for maximum contrast
        dynamic_palette = 'Set1' 
    elif num_hues <= 20:
        # Use 'tab20' for up to 20 categories
        dynamic_palette = sns.color_palette("tab20", n_colors=num_hues)
    elif num_hues > 20:
        # If there are more than 20, we use a different qualitative map like 'hls'
        # WARNING: If num_hues > 20, consider re-binning var2 as visualization will be cluttered.
        dynamic_palette = sns.color_palette("hls", n_colors=num_hues)
    
    
    plt.figure(figsize=(15,8))
    
    # 3. Plot the data using the default column names:
    ax = sns.barplot(
        data=plot_data, 
        x=col_x,
        y='Default Risk Rate (%)',
        hue=col_hue,
        palette=dynamic_palette
    )
    
    plt.title(f"Default Risk Rate (%) for {col_x} by {col_hue}")
    plt.xlabel(col_x)
    plt.ylabel("Default Rate Percentage")
    # You must manually rename the legend title if you don't use .columns
    plt.legend(title=col_hue) 
    plt.xticks(rotation=45, ha='right')
    plt.tight_layout()
    plt.show()

cat_cat_plot(new_app_df,'NAME_EDUCATION_TYPE', 'NAME_INCOME_TYPE')

bivariate_decision_support_risk_plot(new_app_df,'NAME_EDUCATION_TYPE', 'NAME_INCOME_TYPE')

cat_cat_plot(new_app_df,'NAME_FAMILY_STATUS','CODE_GENDER')

bivariate_decision_support_risk_plot(new_app_df, 'NAME_FAMILY_STATUS', 'CODE_GENDER')

cat_cat_plot(new_app_df,'NAME_HOUSING_TYPE','NAME_CONTRACT_TYPE')

bivariate_decision_support_risk_plot(new_app_df,'NAME_HOUSING_TYPE', 'NAME_CONTRACT_TYPE')

cat_cat_plot(new_app_df,'NAME_INCOME_TYPE','OCCUPATION_TYPE')

bivariate_decision_support_risk_plot(new_app_df,'NAME_INCOME_TYPE', 'OCCUPATION_TYPE')

cat_cat_plot(new_app_df, 'FLAG_OWN_CAR','CNT_CHILDREN_OBJ')

bivariate_decision_support_risk_plot(new_app_df,'FLAG_OWN_CAR', 'CNT_CHILDREN_OBJ')

cat_cat_plot(new_app_df, 'FLAG_OWN_REALTY','CNT_FAM_MEMBERS_OBJ')

bivariate_decision_support_risk_plot(new_app_df,'FLAG_OWN_REALTY','CNT_FAM_MEMBERS_OBJ')

# function to plot categorical,contuinious
def cat_cont_plot(df, var1, var2):
    
    plt.figure(figsize=(15,8))

    # --- Subplot 1: DEFAULTERS (df1) ---
    plt.subplot(1, 2, 1)
    # Pass the column names as strings (var1, var2) to sns.countplot
    ax1 = sns.boxplot(data=df1, x=var1, y=var2, showfliers=False)
    # Pass the column names as strings for the title and labels
    plt.title(f"Distribution of {var1} by {var2} for Defaulters")
    plt.xlabel(var1)
    plt.xticks(rotation=45, ha='right')
    plt.ylabel(var2)
    
    # --- Subplot 2: NON-DEFAULTERS (df0) ---
    plt.subplot(1, 2, 2)
    ax2 = sns.boxplot(data=df0, x=var1, y=var2, showfliers=False)
    plt.title(f"Distribution of {var1} by {var2} for Others")
    plt.xlabel(var1)
    plt.xticks(rotation=45, ha='right')
    plt.ylabel(var2) 

    # Crucial for preventing subplot titles and labels from overlapping
    plt.tight_layout()
    plt.show()

cat_cont_plot(new_app_df,'NAME_INCOME_TYPE','AMT_CREDIT')

bivariate_decision_support_risk_plot(new_app_df,'NAME_INCOME_TYPE','CREDIT_BIN_FREQ')

cat_cont_plot(new_app_df,'NAME_EDUCATION_TYPE','AMT_ANNUITY')

bivariate_decision_support_risk_plot(new_app_df,'NAME_EDUCATION_TYPE','ANNUITY_BIN_FREQ')

cat_cont_plot(new_app_df,'OCCUPATION_TYPE','AMT_INCOME_TOTAL')

bivariate_decision_support_risk_plot(new_app_df,'OCCUPATION_TYPE','INCOME_BIN_FREQ')

cat_cont_plot(new_app_df,'NAME_HOUSING_TYPE','AMT_GOODS_PRICE')

bivariate_decision_support_risk_plot(new_app_df,'NAME_HOUSING_TYPE','GOODS_PRICE_BIN_FREQ')

cat_cont_plot(new_app_df,'FLAG_OWN_REALTY','EXT_SOURCE_2')

bivariate_decision_support_risk_plot(new_app_df,'FLAG_OWN_REALTY','EXT_SOURCE_2_BIN_FREQ')

cat_cont_plot(new_app_df,'CNT_FAM_MEMBERS_OBJ','EXT_SOURCE_3')

bivariate_decision_support_risk_plot(new_app_df,'CNT_FAM_MEMBERS_OBJ','EXT_SOURCE_3_BIN_FREQ')

C:\Users\HP\AppData\Local\Temp\ipykernel_10208\3845574278.py:3: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.
  default_rate_pct = df.groupby([col_x, col_hue])['TARGET'].mean()

def cat_cat_cat_plot(df, col_x, col_hue, col_facet, target_col='TARGET'):   
    # The sns.catplot command handles the segmentation by Target and the third variable (col_facet) simultaneously.

    num_hues = df[col_hue].nunique(dropna=False) 

    # 2. Generate the dynamic, high-contrast palette based on hue column
    if num_hues <= 10:
        dynamic_palette = 'Set1' 
    elif num_hues <= 20:
        dynamic_palette = sns.color_palette("tab20", n_colors=num_hues)
    else:
        # Use 'hls' for a larger number, but note visualization will be cluttered
        dynamic_palette = sns.color_palette("hls", n_colors=num_hues)
    
    # 2. Use Seaborn's catplot for efficient faceting and grouping
    g = sns.catplot(
        data=df, # Use the entire DataFrame (no need for df0/df1 split)
        x=col_x,
        hue=col_hue,
        col=target_col,  # Separates Defaulters (1) vs Non-Defaulters (0) side-by-side
        row=col_facet,  # CREATES A NEW ROW OF PLOTS FOR EACH UNIQUE VALUE OF THE THIRD VARIABLE
        kind="count",
        aspect=2.0,  # Stretches the plot horizontally
        height=4,
        palette=dynamic_palette,
        sharey=True,  # All plots share the same Y-axis scale (Count)
        legend_out=True
    )

    # 3. Clean up the plot appearance
    g.set_titles(row_template='{row_name} Group', col_template='Target = {col_name}')
    g.set_axis_labels(col_x, "Count of Applications")
     
     # FIX: Explicitly iterate over the axes to set x-tick labels using the underlying text. 
    for ax in g.axes.flat:
        # CRITICAL FIX: Force Matplotlib to draw the X-axis labels on this specific axis.
        ax.tick_params(labelbottom=True) 
        
        # FIX for AttributeError: 'list' object has no attribute 'size'. Use len() instead of .size
        if len(ax.get_xticks()) > 0:
            # Get the existing labels (the category names)
            labels = [t.get_text() for t in ax.get_xticklabels()]
            # Re-set them with the rotation and alignment
            ax.set_xticklabels(labels, rotation=45, ha='right')
            
     # g.add_legend(title=col_hue, bbox_to_anchor=(1.02, 0.5), loc='center left')

    plt.suptitle(
        f"Multivariate Count Analysis by {col_x}, {col_hue}, and {col_facet}", 
        y=1.02, # Adjust title position
        fontsize=16
    )
    
    # FIX: Increase bottom margin to ensure rotated X-axis labels are not cut off
    g.fig.subplots_adjust(bottom=0.25, right=0.85, hspace=0.7) 
    plt.show()

def multivariate_decision_support_risk_plot(df, col_x, col_hue, col_facet, target_col='TARGET'):
    # 1. Group by all three columns and calculate the mean of TARGET (the Default Rate)
    default_rate_pct = df.groupby([col_x, col_hue, col_facet])[target_col].mean()
    # Convert mean (e.g., 0.08) to percentage (e.g., 8.0) and reset index for plotting
    plot_data = (default_rate_pct * 100).round(2).reset_index()
    plot_data.columns = [col_x, col_hue, col_facet, 'Default Risk Rate (%)']

    num_hues = df[col_hue].nunique(dropna=False) 

    # 2. Generate the dynamic, high-contrast palette based on hue column
    if num_hues <= 10:
        dynamic_palette = 'Set1' 
    elif num_hues <= 20:
        dynamic_palette = sns.color_palette("tab20", n_colors=num_hues)
    else:
        # Use 'hls' for a larger number, but note visualization will be cluttered
        dynamic_palette = sns.color_palette("hls", n_colors=num_hues)
    
    # 3. Use catplot to create a faceted bar plot
    g = sns.catplot(
        data=plot_data,
        x=col_x,
        y='Default Risk Rate (%)', # Plot the pre-calculated percentage
        hue=col_hue,
        col=col_facet,  # The third variable creates the facets (sub-plots)
        kind="bar",
        col_wrap=2,
        aspect=1.5,
        height=6,
        palette=dynamic_palette,
        legend_out=True
    )

    # 4. Clean up the plot appearance
    g.set_axis_labels(col_x, "Default Risk Rate Percentage")
    g.set_titles(col_template='{col_name} Group')
    
     # FIX: Explicitly iterate over the axes to set x-tick labels using the underlying text. 
    for ax in g.axes.flat:
        # CRITICAL FIX: Force Matplotlib to draw the X-axis labels on this specific axis.
        ax.tick_params(labelbottom=True) 
        
        # FIX for AttributeError: 'list' object has no attribute 'size'. Use len() instead of .size
        if len(ax.get_xticks()) > 0:
            # Get the existing labels (the category names)
            labels = [t.get_text() for t in ax.get_xticklabels()]
            # Re-set them with the rotation and alignment
            ax.set_xticklabels(labels, rotation=45, ha='right')
            
    # g.add_legend(title=col_hue, bbox_to_anchor=(1.02, 0.5), loc='center left')

    plt.suptitle(
        f"Multivariate Default Rate (%) by {col_x}, {col_hue}, and {col_facet}", 
        y=1.02,
        fontsize=16
    )
    # 6. Keep increased bottom margin and vertical spacing (hspace) between rows
    g.fig.subplots_adjust(bottom=0.30, right=0.85, hspace=0.75) 
    plt.show()

cat_cat_cat_plot(new_app_df,'NAME_INCOME_TYPE','FLAG_OWN_REALTY','NAME_EDUCATION_TYPE')

multivariate_decision_support_risk_plot(new_app_df,'NAME_INCOME_TYPE','FLAG_OWN_REALTY','NAME_EDUCATION_TYPE')

cat_cat_cat_plot(new_app_df,'CNT_CHILDREN_OBJ','FLAG_OWN_CAR','NAME_EDUCATION_TYPE')

multivariate_decision_support_risk_plot(new_app_df,'CNT_CHILDREN_OBJ','FLAG_OWN_CAR','NAME_EDUCATION_TYPE')

def cat_cat_cont_plot(df, col_x, col_hue, col_y, target_col='TARGET'):
    # 1. Use catplot with kind="box" and set 'col' to target_col for faceting
    g = sns.catplot(
        data=df,
        x=col_x,
        y=col_y,
        hue=col_hue,
        col=target_col,    # CRITICAL CHANGE: Facet by TARGET (0 and 1)
        kind="box",  
        col_wrap=2,        # Ensures the two TARGET plots sit side-by-side
        aspect=1.5,
        height=6,
        legend_out=True,
        sharey=True        # Ensure both plots share the same Y-axis scale
    )

    # 2. Clean up the plot appearance
    g.set_axis_labels(col_x, f"Distribution of {col_y}")
    g.set_titles(col_template='Target = {col_name}') # Label facets as Target = 0 and Target = 1
    
    # Set X-tick labels
    for ax in g.axes.flat:
        ax.tick_params(labelbottom=True) 
        if len(ax.get_xticks()) > 0:
            labels = [t.get_text() for t in ax.get_xticklabels()]
            ax.set_xticklabels(labels, rotation=45, ha='right')

    # Add legend and title
    #g.add_legend(title=col_hue, bbox_to_anchor=(1.02, 0.5), loc='center left')
    plt.suptitle(
        f"Distribution of {col_y} by {col_x}, {col_hue}, and {target_col}", 
        y=1.02, 
        fontsize=16
    )
    g.fig.subplots_adjust(bottom=0.30, right=0.85) 
    plt.show()

cat_cat_cont_plot(new_app_df,'NAME_INCOME_TYPE','FLAG_OWN_REALTY','AMT_CREDIT')

multivariate_decision_support_risk_plot(new_app_df,'NAME_INCOME_TYPE','FLAG_OWN_REALTY','CREDIT_BIN_FREQ')

cat_cat_cont_plot(new_app_df,'NAME_EDUCATION_TYPE','FLAG_OWN_CAR','AMT_ANNUITY')

multivariate_decision_support_risk_plot(new_app_df,'NAME_EDUCATION_TYPE','FLAG_OWN_CAR','ANNUITY_BIN_FREQ')

#defining function for continious-continous columns
def cont_cont_plot(df, var1, var2):
    
    plt.figure(figsize=(15,8))

    # --- Subplot 1: DEFAULTERS (df1) ---
    plt.subplot(1, 2, 1)
    # Pass the column names as strings (var1, var2) to sns.countplot
    ax1 = sns.scatterplot(data=df1, x=var1, y=var2)
    # Pass the column names as strings for the title and labels
    plt.title(f"Correlation of {var1} by {var2} for Defaulters")
    plt.xlabel(var1)
    plt.ylabel(var2)
    
    # --- Subplot 2: NON-DEFAULTERS (df0) ---
    plt.subplot(1, 2, 2)
    ax2 = sns.scatterplot(data=df0, x=var1, y=var2)
    plt.title(f"Correlation of {var1} by {var2} for Others")
    plt.xlabel(var1)
    plt.ylabel(var2) 

    # Crucial for preventing subplot titles and labels from overlapping
    plt.tight_layout()
    plt.show()

#function to calc correlation
def calc_correlation(df,col1,col2):
    # Correlation for Repayers (T=0)
    corr_repayers = df0[col1].corr(df0[col2])   
    # Correlation for Defaulters (T=1)
    corr_defaulters = df1[col1].corr(df1[col2])

    print(f"--- Correlation for {col1} vs. {col2} ---")
    print(f"Repayers (T=0) Correlation: {corr_repayers:.4f}")
    print(f"Defaulters (T=1) Correlation: {corr_defaulters:.4f}")

    return corr_repayers, corr_defaulters

cont_cont_plot(new_app_df,'AMT_CREDIT','AMT_ANNUITY')

rep_corr, def_corr = calc_correlation(new_app_df,'AMT_CREDIT', 'AMT_ANNUITY')

cont_cont_plot(new_app_df,'AMT_CREDIT','AMT_GOODS_PRICE')

rep_corr, def_corr = calc_correlation(new_app_df,'AMT_CREDIT', 'AMT_GOODS_PRICE')

cont_cont_plot(new_app_df,'AMT_CREDIT','AMT_INCOME_TOTAL')

# split data by target 
df0 = new_app_df[new_app_df['TARGET'] == 0]
df1 = new_app_df[new_app_df['TARGET'] == 1]

#compute corr
corr0 = df0.corr(numeric_only=True)
corr1 = df1.corr(numeric_only=True)

def get_top_corr(corr, n=10):
    # take upper triangle only (no duplicates)
    c = corr.where(np.triu(np.ones(corr.shape), k=1).astype(bool))
    c = c.stack().sort_values(ascending=False)
    return c.head(n)

top10_target1 = get_top_corr(corr1, 10)
top10_target0 = get_top_corr(corr0, 10)

print("\nTOP 10 CORRELATIONS — Target = 1")
print(top10_target1)

print("\nTOP 10 CORRELATIONS — Target = 0")
print(top10_target0)

TOP 10 CORRELATIONS — Target = 1
OBS_30_CNT_SOCIAL_CIRCLE  OBS_60_CNT_SOCIAL_CIRCLE    0.998270
BASEMENTAREA_AVG          BASEMENTAREA_MEDI           0.998252
COMMONAREA_AVG            COMMONAREA_MEDI             0.998204
NONLIVINGAPARTMENTS_AVG   NONLIVINGAPARTMENTS_MEDI    0.998116
YEARS_BUILD_AVG           YEARS_BUILD_MEDI            0.998115
FLOORSMIN_AVG             FLOORSMIN_MEDI              0.997826
LIVINGAPARTMENTS_AVG      LIVINGAPARTMENTS_MEDI       0.997721
NONLIVINGAPARTMENTS_MODE  NONLIVINGAPARTMENTS_MEDI    0.997076
ENTRANCES_AVG             ENTRANCES_MEDI              0.996697
LIVINGAREA_AVG            LIVINGAREA_MEDI             0.996050
dtype: float64

TOP 10 CORRELATIONS — Target = 0
YEARS_BUILD_AVG           YEARS_BUILD_MEDI            0.998523
OBS_30_CNT_SOCIAL_CIRCLE  OBS_60_CNT_SOCIAL_CIRCLE    0.998510
FLOORSMIN_AVG             FLOORSMIN_MEDI              0.997244
ENTRANCES_AVG             ENTRANCES_MEDI              0.996913
ELEVATORS_AVG             ELEVATORS_MEDI              0.996747
COMMONAREA_AVG            COMMONAREA_MEDI             0.996119
LIVINGAREA_AVG            LIVINGAREA_MEDI             0.995763
APARTMENTS_AVG            APARTMENTS_MEDI             0.995321
BASEMENTAREA_AVG          BASEMENTAREA_MEDI           0.994160
LIVINGAPARTMENTS_AVG      LIVINGAPARTMENTS_MEDI       0.993889
dtype: float64

# load dataset
prev_df = pd.read_csv('previous_application.csv')

#read first 5 rows
prev_df.head(5)

#read last 5 rows
prev_df.tail(5)

prev_df.shape

prev_df.columns.values

prev_df.dtypes

pd.options.display.float_format = '{:,.2f}'.format
prev_df.describe()

#Count null values per column
null_count = prev_df.isnull().sum()
#Total number of rows
total_count = prev_df.shape[0]
# Calculate percentage of nulls per column and round to 2 decimal places
null_pct2 = ((null_count / total_count) * 100).round(2)
#Filter only columns with missing values (makes plot cleaner)
null_pct2 = null_pct2[null_pct2 > 0].sort_values(ascending=False)

#Plot using pointplot
plt.figure(figsize=(16,5))
sns.pointplot(x=null_pct2.index, y=null_pct2.values)
plt.xticks(rotation=90)
plt.xlabel("Columns")
plt.ylabel("Missing Values (%)")
plt.title("Percentage of Null Values per Column")
plt.show()

# create a copy of base data
new_prev_df = prev_df.copy()

new_prev_df.head(5)

#check for missing values in percentage for each col
null_pct2.sort_values(ascending=True)

#get column names with more than 50% missing
cols_50 = null_pct2[null_pct2 >= 50].index.tolist()
cols_50

['RATE_INTEREST_PRIMARY',
 'RATE_INTEREST_PRIVILEGED',
 'AMT_DOWN_PAYMENT',
 'RATE_DOWN_PAYMENT']

#remove the colums with more than 50% missing

new_prev_df.drop(cols_50, axis=1, inplace=True)
new_prev_df.shape

(1670214, 33)

new_prev_df.drop('NAME_TYPE_SUITE', axis=1, inplace=True)
new_prev_df.shape

(1670214, 32)

new_prev_df['NFLAG_INSURED_ON_APPROVAL'].value_counts()

new_prev_df['NFLAG_INSURED_ON_APPROVAL'].astype(object)

0         0.00
1         1.00
2         1.00
3         1.00
4          NaN
          ... 
1670209   0.00
1670210   0.00
1670211   0.00
1670212   1.00
1670213   0.00
Name: NFLAG_INSURED_ON_APPROVAL, Length: 1670214, dtype: object

#fill null values with NA (third category)
new_prev_df['NFLAG_INSURED_ON_APPROVAL'].fillna('NA', inplace=True)

# check to see any null valus remaining
new_prev_df['NFLAG_INSURED_ON_APPROVAL'].isna().sum()

np.int64(0)

new_prev_df['PRODUCT_COMBINATION'].fillna(new_prev_df['PRODUCT_COMBINATION'].mode()[0], inplace=True)

#check to see any null values remaining
new_prev_df['PRODUCT_COMBINATION'].isna().sum()

np.int64(0)

PLACEHOLDER_CODE = 365243

# 1. Create a boolean Series: True if the column contains the placeholder
#    (Using .any() checks if ANY value in the column is equal to the code)
cols_with_placeholder = (new_prev_df == PLACEHOLDER_CODE).any()

# 2. Filter the index to get the list of column names
placeholder_cols = cols_with_placeholder[cols_with_placeholder].index.tolist()
placeholder_cols

['SK_ID_CURR',
 'DAYS_FIRST_DRAWING',
 'DAYS_FIRST_DUE',
 'DAYS_LAST_DUE_1ST_VERSION',
 'DAYS_LAST_DUE',
 'DAYS_TERMINATION']

cols_to_convert = ['DAYS_FIRST_DRAWING','DAYS_FIRST_DUE','DAYS_LAST_DUE_1ST_VERSION','DAYS_LAST_DUE','DAYS_TERMINATION']

for col in cols_to_convert:
    # Action: Modifies the column directly and returns None
    new_prev_df[col].replace(365243, np.nan, inplace=True)

num_cols = ['AMT_ANNUITY', 'CNT_PAYMENT', 'AMT_GOODS_PRICE', 'DAYS_FIRST_DUE', 
            'DAYS_TERMINATION', 'DAYS_FIRST_DRAWING','DAYS_LAST_DUE_1ST_VERSION','DAYS_LAST_DUE']

for col in num_cols:
    # Ensure you are calculating the median *excluding* the NaNs
    median_val = new_prev_df[col].median()
    new_prev_df[col].fillna(median_val, inplace=True)

#check entire dataframe for null values 
new_prev_df.isnull().sum()

SK_ID_PREV                     0
SK_ID_CURR                     0
NAME_CONTRACT_TYPE             0
AMT_ANNUITY                    0
AMT_APPLICATION                0
AMT_CREDIT                     1
AMT_GOODS_PRICE                0
WEEKDAY_APPR_PROCESS_START     0
HOUR_APPR_PROCESS_START        0
FLAG_LAST_APPL_PER_CONTRACT    0
NFLAG_LAST_APPL_IN_DAY         0
NAME_CASH_LOAN_PURPOSE         0
NAME_CONTRACT_STATUS           0
DAYS_DECISION                  0
NAME_PAYMENT_TYPE              0
CODE_REJECT_REASON             0
NAME_CLIENT_TYPE               0
NAME_GOODS_CATEGORY            0
NAME_PORTFOLIO                 0
NAME_PRODUCT_TYPE              0
CHANNEL_TYPE                   0
SELLERPLACE_AREA               0
NAME_SELLER_INDUSTRY           0
CNT_PAYMENT                    0
NAME_YIELD_GROUP               0
PRODUCT_COMBINATION            0
DAYS_FIRST_DRAWING             0
DAYS_FIRST_DUE                 0
DAYS_LAST_DUE_1ST_VERSION      0
DAYS_LAST_DUE                  0
DAYS_TERMINATION               0
NFLAG_INSURED_ON_APPROVAL      0
dtype: int64

new_prev_df['AMT_CREDIT'].fillna(new_prev_df['AMT_CREDIT'].median(), inplace=True)

#recheck entire dataframe for null values 
new_prev_df.isnull().sum()

SK_ID_PREV                     0
SK_ID_CURR                     0
NAME_CONTRACT_TYPE             0
AMT_ANNUITY                    0
AMT_APPLICATION                0
AMT_CREDIT                     0
AMT_GOODS_PRICE                0
WEEKDAY_APPR_PROCESS_START     0
HOUR_APPR_PROCESS_START        0
FLAG_LAST_APPL_PER_CONTRACT    0
NFLAG_LAST_APPL_IN_DAY         0
NAME_CASH_LOAN_PURPOSE         0
NAME_CONTRACT_STATUS           0
DAYS_DECISION                  0
NAME_PAYMENT_TYPE              0
CODE_REJECT_REASON             0
NAME_CLIENT_TYPE               0
NAME_GOODS_CATEGORY            0
NAME_PORTFOLIO                 0
NAME_PRODUCT_TYPE              0
CHANNEL_TYPE                   0
SELLERPLACE_AREA               0
NAME_SELLER_INDUSTRY           0
CNT_PAYMENT                    0
NAME_YIELD_GROUP               0
PRODUCT_COMBINATION            0
DAYS_FIRST_DRAWING             0
DAYS_FIRST_DUE                 0
DAYS_LAST_DUE_1ST_VERSION      0
DAYS_LAST_DUE                  0
DAYS_TERMINATION               0
NFLAG_INSURED_ON_APPROVAL      0
dtype: int64

count_cols = ['DAYS_FIRST_DRAWING','DAYS_FIRST_DUE','DAYS_LAST_DUE_1ST_VERSION','DAYS_LAST_DUE','DAYS_TERMINATION']

for col in count_cols:
    new_prev_df[col] = new_prev_df[col].astype('int64')

new_prev_df.describe()

pos_days_cols = ['DAYS_FIRST_DRAWING', 'DAYS_FIRST_DUE', 'DAYS_LAST_DUE_1ST_VERSION', 'DAYS_LAST_DUE', 'DAYS_TERMINATION','DAYS_DECISION','SELLERPLACE_AREA']
# concvert to absolute values
for col in pos_days_cols:
    new_prev_df[col]= new_prev_df[col].abs()

outlier_cols = ['AMT_ANNUITY','AMT_APPLICATION','AMT_CREDIT','AMT_GOODS_PRICE']

def find_outliers(df, col):
    Q1 = df[col].quantile(0.25)
    Q3 = df[col].quantile(0.75)
    IQR = Q3 - Q1
    lower = Q1 - 1.5 * IQR
    upper = Q3 + 1.5 * IQR
    outliers = df[(df[col] < lower) | (df[col] > upper)]
    return len(outliers), lower, upper

# Summary
outlier_summary = []
for col in outlier_cols:
    n_out, low, up = find_outliers(new_prev_df, col)
    pct = n_out / len(new_prev_df) * 100
    outlier_summary.append({
        'Column': col,
        'Outliers': n_out,
        '%': round(pct, 2),
        'Lower': round(low, 2),
        'Upper': round(up, 2)
    })
    
summary_df = pd.DataFrame(outlier_summary)
print(summary_df)

            Column  Outliers     %       Lower      Upper
0      AMT_ANNUITY    162620  9.74   -6,368.30  30,739.42
1  AMT_APPLICATION    208019 12.45 -223,740.00 422,820.00
2       AMT_CREDIT    179989 10.78 -264,226.50 504,805.50
3  AMT_GOODS_PRICE    236035 14.13 -101,857.50 349,762.50

fig, axes = plt.subplots(2, 2, figsize=(8,6))
axes = axes.ravel()

for i, col in enumerate(outlier_cols):
    sns.boxplot(data=new_prev_df, y=col, ax=axes[i], color='lightblue')
    axes[i].set_title(col)
    axes[i].grid(True, alpha=0.3)

plt.tight_layout()
plt.show()

#capping
# List of columns to cap
cap_cols = [
    'AMT_ANNUITY','AMT_APPLICATION','AMT_CREDIT','AMT_GOODS_PRICE'
]

# Cap using IQR upper bound
for col in cap_cols:
    Q1 = new_prev_df[col].quantile(0.25)
    Q3 = new_prev_df[col].quantile(0.75)
    IQR = Q3 - Q1
    upper = Q3 + 1.5 * IQR
    new_prev_df[col] = new_prev_df[col].clip(upper=upper)

fig, axes = plt.subplots(2, 2, figsize=(8,6))
axes = axes.ravel()

for i, col in enumerate(outlier_cols):
    sns.boxplot(data=new_prev_df, y=col, ax=axes[i], color='lightblue')
    axes[i].set_title(col)
    axes[i].grid(True, alpha=0.3)

plt.tight_layout()
plt.show()

### NOTE: THIS TAKES ABOUT 20 MINS TO RUN 

import pandas as pd
import numpy as np

import pandas as pd
import numpy as np

# ============================================
# 1. Columns you specifically want
# ============================================

prev_keep = [
    'SK_ID_CURR',
    'AMT_ANNUITY',
    'AMT_APPLICATION',
    'AMT_CREDIT',
    'AMT_GOODS_PRICE',
    'CNT_PAYMENT',
    'DAYS_DECISION',
    'SELLERPLACE_AREA',
    'NAME_CONTRACT_STATUS',
    'CODE_REJECT_REASON',
    'NAME_CASH_LOAN_PURPOSE',
    'PRODUCT_COMBINATION',
    'NFLAG_INSURED_ON_APPROVAL'
]

prev_small = new_prev_df[prev_keep].copy()

# ============================================
# 2. NUMERIC aggregation → median
# ============================================

num_cols = [
    'AMT_ANNUITY',
    'AMT_APPLICATION',
    'AMT_CREDIT',
    'AMT_GOODS_PRICE',
    'CNT_PAYMENT',
    'DAYS_DECISION',
    'SELLERPLACE_AREA'
]

prev_num_agg = (
    prev_small.groupby('SK_ID_CURR')[num_cols]
    .median()
    .reset_index()
)

# ============================================
# 3. CATEGORICAL aggregation → mode
# ============================================

cat_cols = [
    'NAME_CONTRACT_STATUS',
    'CODE_REJECT_REASON',
    'NAME_CASH_LOAN_PURPOSE',
    'PRODUCT_COMBINATION',
    'NFLAG_INSURED_ON_APPROVAL'
]

def mode_or_unknown(x):
    m = x.mode()
    return m.iloc[0] if not m.empty else 'UNKNOWN'

prev_cat_agg = (
    prev_small.groupby('SK_ID_CURR')[cat_cols]
    .agg(mode_or_unknown)
    .reset_index()
)

# ============================================
# 4. Combine numeric + categorical
# ============================================

prev_agg = prev_num_agg.merge(prev_cat_agg, on='SK_ID_CURR', how='left')

# ============================================
# 5. Rename columns to avoid collisions
# ============================================

prev_agg.rename(columns=lambda c: f"PREV_{c}" if c != 'SK_ID_CURR' else c,
                inplace=True)

# ============================================
# 6. Merge with application data
# ============================================

merged_df = new_app_df.merge(prev_agg, on='SK_ID_CURR', how='left')

print("Final merged_df shape:", merged_df.shape)

Final merged_df shape: (307511, 94)

# check the records to see the new merged df
merged_df.head(5)

#Count null values per column
null_count = merged_df.isnull().sum()
#Total number of rows
total_count = merged_df.shape[0]
# Calculate percentage of nulls and round to 2 decimal places
null_pct3 = ((null_count / total_count) * 100).round(2)
#Filter only columns with missing values (makes plot cleaner)
null_pct3 = null_pct3[null_pct3 > 0].sort_values(ascending=False)

#Plot using pointplot
plt.figure(figsize=(16,5))
sns.pointplot(x=null_pct3.index, y=null_pct3.values)
plt.xticks(rotation=90)
plt.xlabel("Columns")
plt.ylabel("Missing Values (%)")
plt.title("Percentage of Null Values per Column")
plt.show()

#check for missing value in percentage
null_pct3.sort_values(ascending=True)

prev_cols = ['PREV_DAYS_DECISION','PREV_AMT_CREDIT','PREV_AMT_APPLICATION','PREV_CNT_PAYMENT','PREV_AMT_ANNUITY','PREV_AMT_GOODS_PRICE','PREV_SELLERPLACE_AREA']

for col in prev_cols:
    merged_df[col].fillna(0, inplace=True)

merged_df['YEARS_EMPLOYED'].fillna(merged_df['YEARS_EMPLOYED'].median(),inplace=True)

cat_col = ['PREV_NAME_CONTRACT_STATUS','PREV_NFLAG_INSURED_ON_APPROVAL','PREV_CODE_REJECT_REASON','PREV_NAME_CASH_LOAN_PURPOSE','PREV_PRODUCT_COMBINATION']

for col in cat_col:
    merged_df[col].fillna('NA', inplace=True)

#check for remaining missing values
missing_count = merged_df.isnull().sum()
missing_data = missing_count[missing_count > 0]
missing_data

CODE_GENDER    4
dtype: int64

merged_df['CODE_GENDER'].fillna(merged_df['CODE_GENDER'].mode()[0],inplace=True)

#check again
missing_count = merged_df.isnull().sum()
missing_data = missing_count[missing_count > 0]
missing_data

Series([], dtype: int64)

# Define the number of quantiles/bins (e.g., 4 quartiles)
N_BINS = 4 

# Create the new binned column
merged_df['PREV_ANNUITY_BIN_FREQ'] = pd.qcut(
    merged_df['PREV_AMT_ANNUITY'], 
    q=N_BINS, 
    # Optional: name your categories (e.g., Low, Medium, High)
    labels=['Q1_Low', 'Q2_MidLow', 'Q3_MidHigh', 'Q4_High'], 
    duplicates='drop' # Handles cases where values are identical at the cutoff
)

# Check the results: notice the counts are much closer to each other
merged_df['PREV_ANNUITY_BIN_FREQ'].value_counts().sort_index()

PREV_ANNUITY_BIN_FREQ
Q1_Low         76878
Q2_MidLow     125809
Q3_MidHigh     27946
Q4_High        76878
Name: count, dtype: int64

# Define the number of quantiles/bins (e.g., 4 quartiles)
N_BINS = 4 

# Create the new binned column
merged_df['PREV_APPLICATION_BIN_FREQ'] = pd.qcut(
    merged_df['PREV_AMT_APPLICATION'], 
    q=N_BINS, 
    # Optional: name your categories (e.g., Low, Medium, High)
    labels=['Q1_Low', 'Q2_MidLow', 'Q3_MidHigh', 'Q4_High'], 
    duplicates='drop' # Handles cases where values are identical at the cutoff
)

# Check the results: notice the counts are much closer to each other
merged_df['PREV_APPLICATION_BIN_FREQ'].value_counts().sort_index()

PREV_APPLICATION_BIN_FREQ
Q1_Low        76889
Q2_MidLow     76875
Q3_MidHigh    76871
Q4_High       76876
Name: count, dtype: int64

# Define the number of quantiles/bins (e.g., 4 quartiles)
N_BINS = 4 

# Create the new binned column
merged_df['PREV_CREDIT_BIN_FREQ'] = pd.qcut(
    merged_df['PREV_AMT_CREDIT'], 
    q=N_BINS, 
    # Optional: name your categories (e.g., Low, Medium, High)
    labels=['Q1_Low', 'Q2_MidLow', 'Q3_MidHigh', 'Q4_High'], 
    duplicates='drop' # Handles cases where values are identical at the cutoff
)

# Check the results: notice the counts are much closer to each other
merged_df['PREV_CREDIT_BIN_FREQ'].value_counts().sort_index()

PREV_CREDIT_BIN_FREQ
Q1_Low        76884
Q2_MidLow     76873
Q3_MidHigh    76876
Q4_High       76878
Name: count, dtype: int64

unique_count = merged_df['PREV_CNT_PAYMENT'].nunique()
print(f"Unique values in PREV_CNT_PAYMENT: {unique_count}")

# Create the new binned column using pd.qcut
# This splits the data into 4 equal-sized groups based on the count of previous payments.
N_BINS = 4 

merged_df['PREV_DAYS_DECISION_BIN_FREQ'] = pd.qcut(
    merged_df['PREV_DAYS_DECISION'], 
    q=N_BINS, # Splits the data into 4 bins (quartiles)
    # 4 bins require exactly 4 labels
    labels=['Q1_Low', 'Q2_MidLow', 'Q3_MidHigh', 'Q4_High'], 
    duplicates='drop' 
)

merged_df['PREV_DAYS_DECISION_BIN_FREQ'].value_counts().sort_index()

[Binned Feature Result]
PREV_DAYS_DECISION_BIN_FREQ
Q1_Low        77099
Q2_MidLow     76673
Q3_MidHigh    76914
Q4_High       76825
Name: count, dtype: int64

# Create the new binned column using pd.qcut
# This splits the data into 4 equal-sized groups based on the count of previous payments.
N_BINS = 4 

merged_df['PREV_GOODS_PRICE_BIN_FREQ'] = pd.qcut(
    merged_df['PREV_AMT_GOODS_PRICE'], 
    q=N_BINS, # Splits the data into 4 bins (quartiles)
    # 4 bins require exactly 4 labels
    labels=['Q1_Low', 'Q2_MidLow', 'Q3_MidHigh', 'Q4_High'], 
    duplicates='drop' 
)

merged_df['PREV_GOODS_PRICE_BIN_FREQ'].value_counts().sort_index()

[Binned Feature Result]

PREV_GOODS_PRICE_BIN_FREQ
Q1_Low         76878
Q2_MidLow     127866
Q3_MidHigh     25896
Q4_High        76871
Name: count, dtype: int64

N_BINS = 4 

# --- SIMPLIFIED SOLUTION: Using pd.cut for fixed-width binning ---
# Since pd.qcut was unstable, pd.cut is used to segment the feature based on value range.
# This operation is stable and simple, avoiding all the previous errors.

merged_df['PREV_PAYMENT_BIN_FREQ'] = pd.cut(
    merged_df['PREV_CNT_PAYMENT'],
    bins=N_BINS, # Automatically calculates 4 equally spaced bins between min and max
    labels=['Q1_Low', 'Q2_MidLow', 'Q3_MidHigh', 'Q4_High'], 
    include_lowest=True # Ensures the minimum value is included in the first bin
)

# Check the results
merged_df['PREV_PAYMENT_BIN_FREQ'].value_counts().sort_index()

[Binned Feature Result (pd.cut)]
PREV_PAYMENT_BIN_FREQ
Q1_Low        277787
Q2_MidLow      26932
Q3_MidHigh      2063
Q4_High          729
Name: count, dtype: int64

cat_plot(merged_df,'PREV_NAME_CONTRACT_STATUS')

univariate_decision_support_risk_plot(merged_df,'PREV_NAME_CONTRACT_STATUS')

cat_plot(merged_df,'PREV_NFLAG_INSURED_ON_APPROVAL')

univariate_decision_support_risk_plot(merged_df,'PREV_NFLAG_INSURED_ON_APPROVAL')

cat_plot(merged_df,'PREV_CODE_REJECT_REASON')

univariate_decision_support_risk_plot(merged_df,'PREV_CODE_REJECT_REASON')

cat_plot(merged_df,'PREV_NAME_CASH_LOAN_PURPOSE')

univariate_decision_support_risk_plot(merged_df,'PREV_NAME_CASH_LOAN_PURPOSE')

cat_plot(merged_df,'PREV_PRODUCT_COMBINATION')

univariate_decision_support_risk_plot(merged_df,'PREV_PRODUCT_COMBINATION')

df0 = merged_df[merged_df['TARGET'] == 0]
df1 = merged_df[merged_df['TARGET'] == 1]

cont_plot(merged_df,'PREV_AMT_ANNUITY')

univariate_decision_support_risk_plot(merged_df,'PREV_ANNUITY_BIN_FREQ')

cont_plot(merged_df,'PREV_AMT_APPLICATION')

univariate_decision_support_risk_plot(merged_df,'PREV_APPLICATION_BIN_FREQ')

cont_plot(merged_df,'PREV_AMT_CREDIT')

univariate_decision_support_risk_plot(merged_df,'PREV_CREDIT_BIN_FREQ')

cont_plot(merged_df,'PREV_AMT_GOODS_PRICE')

univariate_decision_support_risk_plot(merged_df,'PREV_GOODS_PRICE_BIN_FREQ')

cont_plot(merged_df,'PREV_CNT_PAYMENT')

univariate_decision_support_risk_plot(merged_df,'PREV_PAYMENT_BIN_FREQ')

cont_plot(merged_df,'PREV_DAYS_DECISION')

univariate_decision_support_risk_plot(merged_df,'PREV_DAYS_DECISION_BIN_FREQ')

cont_plot(merged_df,'PREV_SELLERPLACE_AREA')

df0 = merged_df[merged_df['TARGET']==0]
df1 = merged_df[merged_df['TARGET']==1]

cat_cat_plot(merged_df,'PREV_NAME_CONTRACT_STATUS','NAME_HOUSING_TYPE')

bivariate_decision_support_risk_plot(merged_df,'PREV_NAME_CONTRACT_STATUS','NAME_HOUSING_TYPE')

cat_cat_plot(merged_df,'PREV_NFLAG_INSURED_ON_APPROVAL','OCCUPATION_TYPE')

bivariate_decision_support_risk_plot(merged_df,'PREV_NFLAG_INSURED_ON_APPROVAL','OCCUPATION_TYPE')

cat_cat_plot(merged_df,'PREV_CODE_REJECT_REASON','CODE_GENDER')

bivariate_decision_support_risk_plot(merged_df,'PREV_CODE_REJECT_REASON','CODE_GENDER')

cat_cat_plot(merged_df,'PREV_NAME_CASH_LOAN_PURPOSE','NAME_FAMILY_STATUS')

bivariate_decision_support_risk_plot(merged_df,'PREV_NAME_CASH_LOAN_PURPOSE','NAME_FAMILY_STATUS')

cat_cont_plot(merged_df,'PREV_AMT_ANNUITY','NAME_INCOME_TYPE')

bivariate_decision_support_risk_plot(merged_df,'PREV_ANNUITY_BIN_FREQ','NAME_INCOME_TYPE')

cat_cont_plot(merged_df,'PREV_AMT_APPLICATION','OCCUPATION_TYPE')

bivariate_decision_support_risk_plot(merged_df,'PREV_APPLICATION_BIN_FREQ','NAME_INCOME_TYPE')

cat_cont_plot(merged_df,'PREV_AMT_CREDIT','NAME_FAMILY_STATUS')

bivariate_decision_support_risk_plot(merged_df,'PREV_CREDIT_BIN_FREQ','NAME_FAMILY_STATUS')

cat_cat_cat_plot(merged_df,'PREV_NAME_CONTRACT_STATUS','NAME_HOUSING_TYPE','NAME_INCOME_TYPE')

multivariate_decision_support_risk_plot(merged_df,'PREV_NAME_CONTRACT_STATUS','NAME_HOUSING_TYPE','NAME_INCOME_TYPE')

cat_cat_cat_plot(merged_df,'PREV_NAME_CASH_LOAN_PURPOSE','NAME_FAMILY_STATUS','FLAG_OWN_REALTY')

multivariate_decision_support_risk_plot(merged_df,'PREV_NAME_CASH_LOAN_PURPOSE','NAME_FAMILY_STATUS','FLAG_OWN_REALTY')

cat_cat_cat_plot(merged_df,'PREV_ANNUITY_BIN_FREQ','NAME_INCOME_TYPE','FLAG_OWN_REALTY')

multivariate_decision_support_risk_plot(merged_df,'PREV_ANNUITY_BIN_FREQ','NAME_INCOME_TYPE','FLAG_OWN_REALTY')

cat_cat_cat_plot(merged_df,'PREV_CREDIT_BIN_FREQ','NAME_FAMILY_STATUS','FLAG_OWN_CAR')

multivariate_decision_support_risk_plot(merged_df,'PREV_CREDIT_BIN_FREQ','NAME_FAMILY_STATUS','FLAG_OWN_CAR')

- PREV_AMT_CREDIT - PREV_AMT_GOODS_PRICE

cont_cont_plot(merged_df,'PREV_AMT_CREDIT','PREV_AMT_GOODS_PRICE')

rep_corr, def_corr = calc_correlation(merged_df,'PREV_AMT_CREDIT', 'PREV_AMT_GOODS_PRICE')

--- Correlation for PREV_AMT_CREDIT vs. PREV_AMT_GOODS_PRICE ---
Repayers (T=0) Correlation: 0.8949
Defaulters (T=1) Correlation: 0.8801

# split data by target 
df0 = merged_df[merged_df['TARGET'] == 0]
df1 = merged_df[merged_df['TARGET'] == 1]

#compute corr
corr0 = df0.corr(numeric_only=True)
corr1 = df1.corr(numeric_only=True)

def get_top_corr(corr, n=10):
    # take upper triangle only (no duplicates)
    c = corr.where(np.triu(np.ones(corr.shape), k=1).astype(bool))
    c = c.stack().sort_values(ascending=False)
    return c.head(n)

top10_target1 = get_top_corr(corr1, 10)
top10_target0 = get_top_corr(corr0, 10)

print("\nTOP 10 CORRELATIONS — Target = 1")
print(top10_target1)

print("\nTOP 10 CORRELATIONS — Target = 0")
print(top10_target0)

TOP 10 CORRELATIONS — Target = 1
OBS_30_CNT_SOCIAL_CIRCLE    OBS_60_CNT_SOCIAL_CIRCLE      1.00
AMT_CREDIT                  AMT_GOODS_PRICE               0.98
REGION_RATING_CLIENT        REGION_RATING_CLIENT_W_CITY   0.96
PREV_AMT_APPLICATION        PREV_AMT_CREDIT               0.94
                            PREV_AMT_GOODS_PRICE          0.91
CNT_CHILDREN                CNT_FAM_MEMBERS               0.88
PREV_AMT_CREDIT             PREV_AMT_GOODS_PRICE          0.88
DEF_30_CNT_SOCIAL_CIRCLE    DEF_60_CNT_SOCIAL_CIRCLE      0.87
REG_REGION_NOT_WORK_REGION  LIVE_REGION_NOT_WORK_REGION   0.85
PREV_AMT_ANNUITY            PREV_AMT_GOODS_PRICE          0.84
dtype: float64

TOP 10 CORRELATIONS — Target = 0
OBS_30_CNT_SOCIAL_CIRCLE    OBS_60_CNT_SOCIAL_CIRCLE      1.00
AMT_CREDIT                  AMT_GOODS_PRICE               0.99
REGION_RATING_CLIENT        REGION_RATING_CLIENT_W_CITY   0.95
PREV_AMT_APPLICATION        PREV_AMT_CREDIT               0.94
                            PREV_AMT_GOODS_PRICE          0.92
PREV_AMT_CREDIT             PREV_AMT_GOODS_PRICE          0.89
CNT_CHILDREN                CNT_FAM_MEMBERS               0.88
REG_REGION_NOT_WORK_REGION  LIVE_REGION_NOT_WORK_REGION   0.86
DEF_30_CNT_SOCIAL_CIRCLE    DEF_60_CNT_SOCIAL_CIRCLE      0.86
PREV_AMT_ANNUITY            PREV_AMT_GOODS_PRICE          0.85
dtype: float64

	SK_ID_CURR	TARGET	NAME_CONTRACT_TYPE	CODE_GENDER	FLAG_OWN_CAR	FLAG_OWN_REALTY	AMT_INCOME_TOTAL	AMT_CREDIT	AMT_ANNUITY	AMT_GOODS_PRICE	NAME_TYPE_SUITE	NAME_INCOME_TYPE	NAME_EDUCATION_TYPE	NAME_FAMILY_STATUS	NAME_HOUSING_TYPE	REGION_POPULATION_RELATIVE	DAYS_BIRTH	DAYS_EMPLOYED	DAYS_REGISTRATION	DAYS_ID_PUBLISH	OWN_CAR_AGE	FLAG_MOBIL	FLAG_EMP_PHONE	FLAG_WORK_PHONE	FLAG_CONT_MOBILE	FLAG_PHONE	FLAG_EMAIL	OCCUPATION_TYPE	CNT_FAM_MEMBERS	REGION_RATING_CLIENT	REGION_RATING_CLIENT_W_CITY	WEEKDAY_APPR_PROCESS_START	HOUR_APPR_PROCESS_START	REG_CITY_NOT_LIVE_CITY	REG_CITY_NOT_WORK_CITY	LIVE_CITY_NOT_WORK_CITY	ORGANIZATION_TYPE	EXT_SOURCE_1	EXT_SOURCE_2	EXT_SOURCE_3	APARTMENTS_AVG	BASEMENTAREA_AVG	YEARS_BEGINEXPLUATATION_AVG	YEARS_BUILD_AVG	COMMONAREA_AVG	ELEVATORS_AVG	ENTRANCES_AVG	FLOORSMAX_AVG	FLOORSMIN_AVG	LANDAREA_AVG	LIVINGAPARTMENTS_AVG	LIVINGAREA_AVG	NONLIVINGAPARTMENTS_AVG	NONLIVINGAREA_AVG	APARTMENTS_MODE	BASEMENTAREA_MODE	YEARS_BEGINEXPLUATATION_MODE	YEARS_BUILD_MODE	COMMONAREA_MODE	ELEVATORS_MODE	ENTRANCES_MODE	FLOORSMAX_MODE	FLOORSMIN_MODE	LANDAREA_MODE	LIVINGAPARTMENTS_MODE	LIVINGAREA_MODE	NONLIVINGAPARTMENTS_MODE	NONLIVINGAREA_MODE	APARTMENTS_MEDI	BASEMENTAREA_MEDI	YEARS_BEGINEXPLUATATION_MEDI	YEARS_BUILD_MEDI	COMMONAREA_MEDI	ELEVATORS_MEDI	ENTRANCES_MEDI	FLOORSMAX_MEDI	FLOORSMIN_MEDI	LANDAREA_MEDI	LIVINGAPARTMENTS_MEDI	LIVINGAREA_MEDI	NONLIVINGAPARTMENTS_MEDI	NONLIVINGAREA_MEDI	FONDKAPREMONT_MODE	HOUSETYPE_MODE	TOTALAREA_MODE	WALLSMATERIAL_MODE	EMERGENCYSTATE_MODE	OBS_30_CNT_SOCIAL_CIRCLE	OBS_60_CNT_SOCIAL_CIRCLE	DAYS_LAST_PHONE_CHANGE	FLAG_DOCUMENT_3	FLAG_DOCUMENT_8	AMT_REQ_CREDIT_BUREAU_HOUR	AMT_REQ_CREDIT_BUREAU_DAY	AMT_REQ_CREDIT_BUREAU_WEEK	AMT_REQ_CREDIT_BUREAU_MON	AMT_REQ_CREDIT_BUREAU_QRT	AMT_REQ_CREDIT_BUREAU_YEAR
307506	456251	0	Cash loans	M	N	N	157500.0	254700.0	27558.0	225000.0	Unaccompanied	Working	Secondary / secondary special	Separated	With parents	0.032561	-9327	-236	-8456.0	-1982	NaN	1	1	0	1	0	0	Sales staff	1.0	1	1	THURSDAY	15	0	0	0	Services	0.145570	0.681632	NaN	0.2021	0.0887	0.9876	0.8300	0.0202	0.22	0.1034	0.6042	0.2708	0.0594	0.1484	0.1965	0.0753	0.1095	0.1008	0.0172	0.9782	0.7125	0.0172	0.0806	0.0345	0.4583	0.0417	0.0094	0.0882	0.0853	0.0	0.0125	0.2040	0.0887	0.9876	0.8323	0.0203	0.22	0.1034	0.6042	0.2708	0.0605	0.1509	0.2001	0.0757	0.1118	reg oper account	block of flats	0.2898	Stone, brick	No	0.0	0.0	-273.0	0	1	NaN	NaN	NaN	NaN	NaN	NaN
307507	456252	0	Cash loans	F	N	Y	72000.0	269550.0	12001.5	225000.0	Unaccompanied	Pensioner	Secondary / secondary special	Widow	House / apartment	0.025164	-20775	365243	-4388.0	-4090	NaN	1	0	0	1	1	0	NaN	1.0	2	2	MONDAY	8	0	0	0	XNA	NaN	0.115992	NaN	0.0247	0.0435	0.9727	0.6260	0.0022	0.00	0.1034	0.0833	0.1250	0.0579	0.0202	0.0257	0.0000	0.0000	0.0252	0.0451	0.9727	0.6406	0.0022	0.0000	0.1034	0.0833	0.1250	0.0592	0.0220	0.0267	0.0	0.0000	0.0250	0.0435	0.9727	0.6310	0.0022	0.00	0.1034	0.0833	0.1250	0.0589	0.0205	0.0261	0.0000	0.0000	reg oper account	block of flats	0.0214	Stone, brick	No	0.0	0.0	0.0	1	0	NaN	NaN	NaN	NaN	NaN	NaN
307508	456253	0	Cash loans	F	N	Y	153000.0	677664.0	29979.0	585000.0	Unaccompanied	Working	Higher education	Separated	House / apartment	0.005002	-14966	-7921	-6737.0	-5150	NaN	1	1	0	1	0	1	Managers	1.0	3	3	THURSDAY	9	0	1	1	School	0.744026	0.535722	0.218859	0.1031	0.0862	0.9816	0.7484	0.0123	0.00	0.2069	0.1667	0.2083	NaN	0.0841	0.9279	0.0000	0.0000	0.1050	0.0894	0.9816	0.7583	0.0124	0.0000	0.2069	0.1667	0.2083	NaN	0.0918	0.9667	0.0	0.0000	0.1041	0.0862	0.9816	0.7518	0.0124	0.00	0.2069	0.1667	0.2083	NaN	0.0855	0.9445	0.0000	0.0000	reg oper account	block of flats	0.7970	Panel	No	6.0	6.0	-1909.0	1	0	1.0	0.0	0.0	1.0	0.0	1.0
307509	456254	1	Cash loans	F	N	Y	171000.0	370107.0	20205.0	319500.0	Unaccompanied	Commercial associate	Secondary / secondary special	Married	House / apartment	0.005313	-11961	-4786	-2562.0	-931	NaN	1	1	0	1	0	0	Laborers	2.0	2	2	WEDNESDAY	9	1	1	0	Business Entity Type 1	NaN	0.514163	0.661024	0.0124	NaN	0.9771	NaN	NaN	NaN	0.0690	0.0417	NaN	NaN	NaN	0.0061	NaN	NaN	0.0126	NaN	0.9772	NaN	NaN	NaN	0.0690	0.0417	NaN	NaN	NaN	0.0063	NaN	NaN	0.0125	NaN	0.9771	NaN	NaN	NaN	0.0690	0.0417	NaN	NaN	NaN	0.0062	NaN	NaN	NaN	block of flats	0.0086	Stone, brick	No	0.0	0.0	-322.0	1	0	0.0	0.0	0.0	0.0	0.0	0.0
307510	456255	0	Cash loans	F	N	N	157500.0	675000.0	49117.5	675000.0	Unaccompanied	Commercial associate	Higher education	Married	House / apartment	0.046220	-16856	-1262	-5128.0	-410	NaN	1	1	1	1	1	0	Laborers	2.0	1	1	THURSDAY	20	0	1	1	Business Entity Type 3	0.734460	0.708569	0.113922	0.0742	0.0526	0.9881	NaN	0.0176	0.08	0.0690	0.3750	NaN	NaN	NaN	0.0791	NaN	0.0000	0.0756	0.0546	0.9881	NaN	0.0178	0.0806	0.0690	0.3750	NaN	NaN	NaN	0.0824	NaN	0.0000	0.0749	0.0526	0.9881	NaN	0.0177	0.08	0.0690	0.3750	NaN	NaN	NaN	0.0805	NaN	0.0000	NaN	block of flats	0.0718	Panel	No	0.0	0.0	-787.0	1	0	0.0	0.0	0.0	2.0	0.0	1.0

	SK_ID_CURR	TARGET	NAME_CONTRACT_TYPE	CODE_GENDER	FLAG_OWN_CAR	FLAG_OWN_REALTY	AMT_INCOME_TOTAL	AMT_CREDIT	AMT_ANNUITY	AMT_GOODS_PRICE	NAME_TYPE_SUITE	NAME_INCOME_TYPE	NAME_EDUCATION_TYPE	NAME_FAMILY_STATUS	NAME_HOUSING_TYPE	REGION_POPULATION_RELATIVE	DAYS_BIRTH	DAYS_EMPLOYED	DAYS_REGISTRATION	DAYS_ID_PUBLISH	OWN_CAR_AGE	FLAG_MOBIL	FLAG_EMP_PHONE	FLAG_WORK_PHONE	FLAG_CONT_MOBILE	FLAG_PHONE	OCCUPATION_TYPE	CNT_FAM_MEMBERS	REGION_RATING_CLIENT	REGION_RATING_CLIENT_W_CITY	WEEKDAY_APPR_PROCESS_START	HOUR_APPR_PROCESS_START	REG_CITY_NOT_WORK_CITY	LIVE_CITY_NOT_WORK_CITY	ORGANIZATION_TYPE	EXT_SOURCE_1	EXT_SOURCE_2	EXT_SOURCE_3	APARTMENTS_AVG	BASEMENTAREA_AVG	YEARS_BEGINEXPLUATATION_AVG	YEARS_BUILD_AVG	COMMONAREA_AVG	ELEVATORS_AVG	ENTRANCES_AVG	FLOORSMAX_AVG	FLOORSMIN_AVG	LANDAREA_AVG	LIVINGAPARTMENTS_AVG	LIVINGAREA_AVG	NONLIVINGAPARTMENTS_AVG	NONLIVINGAREA_AVG	APARTMENTS_MODE	BASEMENTAREA_MODE	YEARS_BEGINEXPLUATATION_MODE	YEARS_BUILD_MODE	COMMONAREA_MODE	ELEVATORS_MODE	ENTRANCES_MODE	FLOORSMAX_MODE	FLOORSMIN_MODE	LANDAREA_MODE	LIVINGAPARTMENTS_MODE	LIVINGAREA_MODE	NONLIVINGAPARTMENTS_MODE	NONLIVINGAREA_MODE	APARTMENTS_MEDI	BASEMENTAREA_MEDI	YEARS_BEGINEXPLUATATION_MEDI	YEARS_BUILD_MEDI	COMMONAREA_MEDI	ELEVATORS_MEDI	ENTRANCES_MEDI	FLOORSMAX_MEDI	FLOORSMIN_MEDI	LANDAREA_MEDI	LIVINGAPARTMENTS_MEDI	LIVINGAREA_MEDI	NONLIVINGAPARTMENTS_MEDI	NONLIVINGAREA_MEDI	FONDKAPREMONT_MODE	HOUSETYPE_MODE	TOTALAREA_MODE	WALLSMATERIAL_MODE	EMERGENCYSTATE_MODE	OBS_30_CNT_SOCIAL_CIRCLE	DEF_30_CNT_SOCIAL_CIRCLE	OBS_60_CNT_SOCIAL_CIRCLE	DEF_60_CNT_SOCIAL_CIRCLE	DAYS_LAST_PHONE_CHANGE	FLAG_DOCUMENT_3	FLAG_DOCUMENT_8	AMT_REQ_CREDIT_BUREAU_HOUR	AMT_REQ_CREDIT_BUREAU_DAY	AMT_REQ_CREDIT_BUREAU_WEEK	AMT_REQ_CREDIT_BUREAU_MON	AMT_REQ_CREDIT_BUREAU_QRT	AMT_REQ_CREDIT_BUREAU_YEAR
0	100002	1	Cash loans	M	N	Y	202500.0	406597.5	24700.5	351000.0	Unaccompanied	Working	Secondary / secondary special	Single / not married	House / apartment	0.018801	-9461	-637	-3648.0	-2120	NaN	1	1	0	1	1	Laborers	1.0	2	2	WEDNESDAY	10	0	0	Business Entity Type 3	0.083037	0.262949	0.139376	0.0247	0.0369	0.9722	0.6192	0.0143	0.00	0.0690	0.0833	0.1250	0.0369	0.0202	0.0190	0.0000	0.0000	0.0252	0.0383	0.9722	0.6341	0.0144	0.0000	0.0690	0.0833	0.1250	0.0377	0.022	0.0198	0.0	0.0	0.0250	0.0369	0.9722	0.6243	0.0144	0.00	0.0690	0.0833	0.1250	0.0375	0.0205	0.0193	0.0000	0.00	reg oper account	block of flats	0.0149	Stone, brick	No	2.0	2.0	2.0	2.0	-1134.0	1	0	0.0	0.0	0.0	0.0	0.0	1.0
1	100003	0	Cash loans	F	N	N	270000.0	1293502.5	35698.5	1129500.0	Family	State servant	Higher education	Married	House / apartment	0.003541	-16765	-1188	-1186.0	-291	NaN	1	1	0	1	1	Core staff	2.0	1	1	MONDAY	11	0	0	School	0.311267	0.622246	NaN	0.0959	0.0529	0.9851	0.7960	0.0605	0.08	0.0345	0.2917	0.3333	0.0130	0.0773	0.0549	0.0039	0.0098	0.0924	0.0538	0.9851	0.8040	0.0497	0.0806	0.0345	0.2917	0.3333	0.0128	0.079	0.0554	0.0	0.0	0.0968	0.0529	0.9851	0.7987	0.0608	0.08	0.0345	0.2917	0.3333	0.0132	0.0787	0.0558	0.0039	0.01	reg oper account	block of flats	0.0714	Block	No	1.0	0.0	1.0	0.0	-828.0	1	0	0.0	0.0	0.0	0.0	0.0	0.0
2	100004	0	Revolving loans	M	Y	Y	67500.0	135000.0	6750.0	135000.0	Unaccompanied	Working	Secondary / secondary special	Single / not married	House / apartment	0.010032	-19046	-225	-4260.0	-2531	26.0	1	1	1	1	1	Laborers	1.0	2	2	MONDAY	9	0	0	Government	NaN	0.555912	0.729567	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	0.0	0.0	0.0	0.0	-815.0	0	0	0.0	0.0	0.0	0.0	0.0	0.0
3	100006	0	Cash loans	F	N	Y	135000.0	312682.5	29686.5	297000.0	Unaccompanied	Working	Secondary / secondary special	Civil marriage	House / apartment	0.008019	-19005	-3039	-9833.0	-2437	NaN	1	1	0	1	0	Laborers	2.0	2	2	WEDNESDAY	17	0	0	Business Entity Type 3	NaN	0.650442	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	2.0	0.0	2.0	0.0	-617.0	1	0	NaN	NaN	NaN	NaN	NaN	NaN
4	100007	0	Cash loans	M	N	Y	121500.0	513000.0	21865.5	513000.0	Unaccompanied	Working	Secondary / secondary special	Single / not married	House / apartment	0.028663	-19932	-3038	-4311.0	-3458	NaN	1	1	0	1	0	Core staff	1.0	2	2	THURSDAY	11	1	1	Religion	NaN	0.322738	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	0.0	0.0	0.0	0.0	-1106.0	0	1	0.0	0.0	0.0	0.0	0.0	0.0

	SK_ID_CURR	TARGET	CNT_CHILDREN	AMT_INCOME_TOTAL	AMT_CREDIT	AMT_ANNUITY	AMT_GOODS_PRICE	REGION_POPULATION_RELATIVE	DAYS_BIRTH	DAYS_EMPLOYED	DAYS_REGISTRATION	DAYS_ID_PUBLISH	OWN_CAR_AGE	FLAG_MOBIL	FLAG_EMP_PHONE	FLAG_WORK_PHONE	FLAG_CONT_MOBILE	FLAG_PHONE	FLAG_EMAIL	CNT_FAM_MEMBERS	REGION_RATING_CLIENT	REGION_RATING_CLIENT_W_CITY	HOUR_APPR_PROCESS_START	REG_REGION_NOT_LIVE_REGION	REG_REGION_NOT_WORK_REGION	LIVE_REGION_NOT_WORK_REGION	REG_CITY_NOT_LIVE_CITY	REG_CITY_NOT_WORK_CITY	LIVE_CITY_NOT_WORK_CITY	EXT_SOURCE_1	EXT_SOURCE_2	EXT_SOURCE_3	APARTMENTS_AVG	BASEMENTAREA_AVG	YEARS_BEGINEXPLUATATION_AVG	YEARS_BUILD_AVG	COMMONAREA_AVG	ELEVATORS_AVG	ENTRANCES_AVG	FLOORSMAX_AVG	FLOORSMIN_AVG	LANDAREA_AVG	LIVINGAPARTMENTS_AVG	LIVINGAREA_AVG	NONLIVINGAPARTMENTS_AVG	NONLIVINGAREA_AVG	APARTMENTS_MODE	BASEMENTAREA_MODE	YEARS_BEGINEXPLUATATION_MODE	YEARS_BUILD_MODE	COMMONAREA_MODE	ELEVATORS_MODE	ENTRANCES_MODE	FLOORSMAX_MODE	FLOORSMIN_MODE	LANDAREA_MODE	LIVINGAPARTMENTS_MODE	LIVINGAREA_MODE	NONLIVINGAPARTMENTS_MODE	NONLIVINGAREA_MODE	APARTMENTS_MEDI	BASEMENTAREA_MEDI	YEARS_BEGINEXPLUATATION_MEDI	YEARS_BUILD_MEDI	COMMONAREA_MEDI	ELEVATORS_MEDI	ENTRANCES_MEDI	FLOORSMAX_MEDI	FLOORSMIN_MEDI	LANDAREA_MEDI	LIVINGAPARTMENTS_MEDI	LIVINGAREA_MEDI	NONLIVINGAPARTMENTS_MEDI	NONLIVINGAREA_MEDI	TOTALAREA_MODE	OBS_30_CNT_SOCIAL_CIRCLE	DEF_30_CNT_SOCIAL_CIRCLE	OBS_60_CNT_SOCIAL_CIRCLE	DEF_60_CNT_SOCIAL_CIRCLE	DAYS_LAST_PHONE_CHANGE	FLAG_DOCUMENT_2	FLAG_DOCUMENT_3	FLAG_DOCUMENT_4	FLAG_DOCUMENT_5	FLAG_DOCUMENT_6	FLAG_DOCUMENT_7	FLAG_DOCUMENT_8	FLAG_DOCUMENT_9	FLAG_DOCUMENT_10	FLAG_DOCUMENT_11	FLAG_DOCUMENT_12	FLAG_DOCUMENT_13	FLAG_DOCUMENT_14	FLAG_DOCUMENT_15	FLAG_DOCUMENT_16	FLAG_DOCUMENT_17	FLAG_DOCUMENT_18	FLAG_DOCUMENT_19	FLAG_DOCUMENT_20	FLAG_DOCUMENT_21	AMT_REQ_CREDIT_BUREAU_HOUR	AMT_REQ_CREDIT_BUREAU_DAY	AMT_REQ_CREDIT_BUREAU_WEEK	AMT_REQ_CREDIT_BUREAU_MON	AMT_REQ_CREDIT_BUREAU_QRT	AMT_REQ_CREDIT_BUREAU_YEAR
count	307,511.00	307,511.00	307,511.00	307,511.00	307,511.00	307,499.00	307,233.00	307,511.00	307,511.00	307,511.00	307,511.00	307,511.00	104,582.00	307,511.00	307,511.00	307,511.00	307,511.00	307,511.00	307,511.00	307,509.00	307,511.00	307,511.00	307,511.00	307,511.00	307,511.00	307,511.00	307,511.00	307,511.00	307,511.00	134,133.00	306,851.00	246,546.00	151,450.00	127,568.00	157,504.00	103,023.00	92,646.00	143,620.00	152,683.00	154,491.00	98,869.00	124,921.00	97,312.00	153,161.00	93,997.00	137,829.00	151,450.00	127,568.00	157,504.00	103,023.00	92,646.00	143,620.00	152,683.00	154,491.00	98,869.00	124,921.00	97,312.00	153,161.00	93,997.00	137,829.00	151,450.00	127,568.00	157,504.00	103,023.00	92,646.00	143,620.00	152,683.00	154,491.00	98,869.00	124,921.00	97,312.00	153,161.00	93,997.00	137,829.00	159,080.00	306,490.00	306,490.00	306,490.00	306,490.00	307,510.00	307,511.00	307,511.00	307,511.00	307,511.00	307,511.00	307,511.00	307,511.00	307,511.00	307,511.00	307,511.00	307,511.00	307,511.00	307,511.00	307,511.00	307,511.00	307,511.00	307,511.00	307,511.00	307,511.00	307,511.00	265,992.00	265,992.00	265,992.00	265,992.00	265,992.00	265,992.00
mean	278,180.52	0.08	0.42	168,797.92	599,026.00	27,108.57	538,396.21	0.02	-16,037.00	63,815.05	-4,986.12	-2,994.20	12.06	1.00	0.82	0.20	1.00	0.28	0.06	2.15	2.05	2.03	12.06	0.02	0.05	0.04	0.08	0.23	0.18	0.50	0.51	0.51	0.12	0.09	0.98	0.75	0.04	0.08	0.15	0.23	0.23	0.07	0.10	0.11	0.01	0.03	0.11	0.09	0.98	0.76	0.04	0.07	0.15	0.22	0.23	0.06	0.11	0.11	0.01	0.03	0.12	0.09	0.98	0.76	0.04	0.08	0.15	0.23	0.23	0.07	0.10	0.11	0.01	0.03	0.10	1.42	0.14	1.41	0.10	-962.86	0.00	0.71	0.00	0.02	0.09	0.00	0.08	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.01	0.00	0.01	0.00	0.00	0.00	0.01	0.01	0.03	0.27	0.27	1.90
std	102,790.18	0.27	0.72	237,123.15	402,490.78	14,493.74	369,446.46	0.01	4,363.99	141,275.77	3,522.89	1,509.45	11.94	0.00	0.38	0.40	0.04	0.45	0.23	0.91	0.51	0.50	3.27	0.12	0.22	0.20	0.27	0.42	0.38	0.21	0.19	0.19	0.11	0.08	0.06	0.11	0.08	0.13	0.10	0.14	0.16	0.08	0.09	0.11	0.05	0.07	0.11	0.08	0.06	0.11	0.07	0.13	0.10	0.14	0.16	0.08	0.10	0.11	0.05	0.07	0.11	0.08	0.06	0.11	0.08	0.13	0.10	0.15	0.16	0.08	0.09	0.11	0.05	0.07	0.11	2.40	0.45	2.38	0.36	826.81	0.01	0.45	0.01	0.12	0.28	0.01	0.27	0.06	0.00	0.06	0.00	0.06	0.05	0.03	0.10	0.02	0.09	0.02	0.02	0.02	0.08	0.11	0.20	0.92	0.79	1.87
min	100,002.00	0.00	0.00	25,650.00	45,000.00	1,615.50	40,500.00	0.00	-25,229.00	-17,912.00	-24,672.00	-7,197.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	1.00	1.00	1.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.01	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	-4,292.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00
25%	189,145.50	0.00	0.00	112,500.00	270,000.00	16,524.00	238,500.00	0.01	-19,682.00	-2,760.00	-7,479.50	-4,299.00	5.00	1.00	1.00	0.00	1.00	0.00	0.00	2.00	2.00	2.00	10.00	0.00	0.00	0.00	0.00	0.00	0.00	0.33	0.39	0.37	0.06	0.04	0.98	0.69	0.01	0.00	0.07	0.17	0.08	0.02	0.05	0.05	0.00	0.00	0.05	0.04	0.98	0.70	0.01	0.00	0.07	0.17	0.08	0.02	0.05	0.04	0.00	0.00	0.06	0.04	0.98	0.69	0.01	0.00	0.07	0.17	0.08	0.02	0.05	0.05	0.00	0.00	0.04	0.00	0.00	0.00	0.00	-1,570.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00
50%	278,202.00	0.00	0.00	147,150.00	513,531.00	24,903.00	450,000.00	0.02	-15,750.00	-1,213.00	-4,504.00	-3,254.00	9.00	1.00	1.00	0.00	1.00	0.00	0.00	2.00	2.00	2.00	12.00	0.00	0.00	0.00	0.00	0.00	0.00	0.51	0.57	0.54	0.09	0.08	0.98	0.76	0.02	0.00	0.14	0.17	0.21	0.05	0.08	0.07	0.00	0.00	0.08	0.07	0.98	0.76	0.02	0.00	0.14	0.17	0.21	0.05	0.08	0.07	0.00	0.00	0.09	0.08	0.98	0.76	0.02	0.00	0.14	0.17	0.21	0.05	0.08	0.07	0.00	0.00	0.07	0.00	0.00	0.00	0.00	-757.00	0.00	1.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	1.00
75%	367,142.50	0.00	1.00	202,500.00	808,650.00	34,596.00	679,500.00	0.03	-12,413.00	-289.00	-2,010.00	-1,720.00	15.00	1.00	1.00	0.00	1.00	1.00	0.00	3.00	2.00	2.00	14.00	0.00	0.00	0.00	0.00	0.00	0.00	0.68	0.66	0.67	0.15	0.11	0.99	0.82	0.05	0.12	0.21	0.33	0.38	0.09	0.12	0.13	0.00	0.03	0.14	0.11	0.99	0.82	0.05	0.12	0.21	0.33	0.38	0.08	0.13	0.13	0.00	0.02	0.15	0.11	0.99	0.83	0.05	0.12	0.21	0.33	0.38	0.09	0.12	0.13	0.00	0.03	0.13	2.00	0.00	2.00	0.00	-274.00	0.00	1.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	0.00	3.00
max	456,255.00	1.00	19.00	117,000,000.00	4,050,000.00	258,025.50	4,050,000.00	0.07	-7,489.00	365,243.00	0.00	0.00	91.00	1.00	1.00	1.00	1.00	1.00	1.00	20.00	3.00	3.00	23.00	1.00	1.00	1.00	1.00	1.00	1.00	0.96	0.85	0.90	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	348.00	34.00	344.00	24.00	0.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	1.00	4.00	9.00	8.00	27.00	261.00	25.00

1.Case Study Introduction¶

1.1 Business Understanding¶

1.2 Business Objectives¶

1.3 Data Understanding¶

2. Data Exploration¶

2.1 Read the data file¶

2.2 Inspect the data frame¶

3. Data Cleaning¶

3.1 Missing Value Imputation¶

3.2 Data Correction¶

3.3 Data Standardization¶

3.4 Outlier Analysis¶

3.5 Feature Binning¶

AMT_CREDIT¶

AMT_ANNUITY¶

AMT_INCOME_TOTAL¶

AMT_GOODS_PRICE¶

AGE¶

YEARS_EMPLOYED¶

CNT_CHILDREN¶

CNT_FAM_MEMBERS¶

EXT_SOURCE_2¶

EXT_SOURCE_3¶

4. Data Analysis¶

4.1 Segmented Univariate Analysis¶

Univariate Categorical Analysis¶

NAME_CONTRACT_TYPE¶

CODE_GENDER¶

OCCUPATION_TYPE¶

NAME_INCOME_TYPE¶

NAME_EDUCATION_TYPE¶

NAME_FAMILY_STATUS¶

NAME_HOUSING_TYPE¶

CNT_CHILDREN¶

CNT_FAM_MEMBERS¶

FLAG_OWN_CAR¶

FLAG_OWN_REALTY¶

REGION_RATING_CLIENT¶

Univariate Numerical Analysis¶

AMT_INCOME_TOTAL¶

AMT_CREDIT¶

AMT_ANNUITY¶

AMT_GOODS_PRICE¶

AGE_YEARS¶

YEARS_EMPLOYED¶

EXT_SOURCE_2¶

EXT_SOURCE_3¶

Univariate Numerical Discrete Analysis¶

4.2 Bivariate Analysis¶

Categorical-categorical columns¶

NAME_EDUCATION_TYPE - NAME_INCOME_TYPE¶

NAME_FAMILY_STATUS - CODE_GENDER¶

NAME_HOUSING_TYPE - NAME_CONTRACT_STATUS¶

NAME_INCOME_TYPE - OCCUPATION_TYPE¶

FLAG_OWN_CAR - CNT_CHILDREN¶

FLAG_OWN_REALTY - CNT_FAM_MEMBERS¶

Categorical-continuos columns¶

NAME_INCOME_TYPE - AMT_CREDIT¶

NAME_EDUCATION_TYPE - AMT_ANNUITY¶

OCCUPATION_TYPE - AMT_INCOME_TOTAL¶

HOUSING_TYPE - AMT_GOODS_PRICE¶

FLAG_OWN_REALTY - EXT_SOURCE_2¶

CNT_FAM_MEMBERS - EXT_SOURCE_3¶

Multivariate Analysis¶

Categorical-categorical-categorical¶

NAME_INCOME_TYPE - FLAG_OWN_REALTY - NAME_EDUCATION_TYPE¶

CNT_CHILDREN - FLAG_OWN_CAR - NAME_EDUCATION_TYPE¶

Categorical - cateogrical - continuos¶

NAME_INCOME_TYPE - FLAG_OWN_REALTY - AMT_CREDIT¶

NAME_EDUCATION_TYPE - FLAG_OWN_CAR - AMT_ANNUITY¶

Continous-continuos columns¶

4.3 Correlation¶

5. PREVIOUS_APPLICATION¶

5.1 Read the Data File¶

5.2 Inspect the Data Frame¶

5.3 Data Imbalance Checking¶

6.Data Cleaning¶

6.1 Missing Value Imputation¶

6.2 Data Type Correction¶

6.3 Data Standardization¶